diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dd74adccf..e35c291e7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -249,6 +249,9 @@ catalogs: nanoid: specifier: 5.1.6 version: 5.1.6 + node-pty: + specifier: ^1.1.0 + version: 1.1.0 ofetch: specifier: ^1.5.1 version: 1.5.1 @@ -3749,6 +3752,21 @@ importers: specifier: ^0.20.20 version: 0.20.20(@types/node@24.12.0)(canvas@3.2.2)(eslint@10.1.0(jiti@2.6.1))(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(rollup@4.60.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + services/computer-use-mcp: + dependencies: + '@modelcontextprotocol/sdk': + specifier: 'catalog:' + version: 1.27.1(@cfworker/json-schema@4.1.1)(zod@4.3.6) + node-pty: + specifier: 'catalog:' + version: 1.1.0 + ws: + specifier: ^8.19.0 + version: 8.20.0(bufferutil@4.1.0)(utf-8-validate@5.0.10) + zod: + specifier: ^4.3.6 + version: 4.3.6 + services/discord-bot: dependencies: '@discordjs/voice': @@ -13066,7 +13084,6 @@ packages: glob@11.1.0: resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==} engines: {node: 20 || >=22} - deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@13.0.0: @@ -14761,6 +14778,9 @@ packages: node-notifier@10.0.1: resolution: {integrity: sha512-YX7TSyDukOZ0g+gmzjB6abKu+hTGvO8+8+gIFDsRCU2t8fLV/P2unmt+LGFaIa4y64aX98Qksa97rgz4vMNeLQ==} + node-pty@1.1.0: + resolution: {integrity: sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==} + node-releases@2.0.27: resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} @@ -29740,6 +29760,10 @@ snapshots: uuid: 8.3.2 which: 2.0.2 + node-pty@1.1.0: + dependencies: + node-addon-api: 7.1.1 + node-releases@2.0.27: {} node-rsa@0.4.2: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e5d586efe..17b4056a8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -111,6 +111,7 @@ catalog: mkcert: ^3.2.0 nano-staged: ^0.9.0 nanoid: 5.1.6 + node-pty: ^1.1.0 ofetch: ^1.5.1 oxc-minify: ^0.121.0 pinia: ^3.0.4 @@ -120,6 +121,8 @@ catalog: superjson: ^2.2.6 tsdown: ^0.21.4 tsx: ^4.21.0 + typescript: ~5.9.3 + typescript-eslint: ^8.51.0 uncrypto: ^0.1.3 unplugin-info: ^1.2.4 unstorage: ^1.17.4 diff --git a/services/computer-use-mcp/FEASIBILITY.md b/services/computer-use-mcp/FEASIBILITY.md new file mode 100644 index 000000000..2336f6418 --- /dev/null +++ b/services/computer-use-mcp/FEASIBILITY.md @@ -0,0 +1,108 @@ +# Feasibility Summary + +This document records the validated state of the AIRI-specific macOS desktop orchestration v1 in `services/computer-use-mcp`. + +## Bottom Line + +The current direction is feasible and materially stronger than the earlier pure-vision-only path. +The validated architecture is now: + +- AIRI keeps the control plane +- `computer-use-mcp` keeps trace, audit, screenshot persistence, policy, and the MCP surface +- the primary execution backend is local macOS window automation +- AIRI desktop handles human approval through native dialogs +- terminal commands run in a controlled background shell runner instead of a Terminal tab script + +That makes the feature an orchestration layer, not just a mouse-clicking demo. + +## What Was Verified + +### 1. The service still fits AIRI's MCP attachment model + +AIRI continues to use the existing stdio MCP bridge through `mcp.json`. +No transport rewrite was required. + +### 2. The service now exposes a tool-first desktop orchestration surface + +Validated surface in this checkout: + +- desktop observation tools +- deterministic app open/focus tools +- background terminal execution tools +- primitive UI interaction tools +- approval / trace / audit helpers + +This is a better fit for AIRI than leading with pure screenshot-driven action selection. + +### 3. Terminal execution is now first-class and auditable + +Validated by tests: + +- commands run in a local background shell process +- non-zero exit codes are returned without throwing away stderr/stdout +- cwd is sticky across calls unless explicitly overridden +- reset clears terminal state + +This gives AIRI a deterministic execution path for many developer workflows without relying on Terminal.app scripting. + +### 4. Native approval now sits in the AIRI desktop layer + +Validated by implementation shape: + +- MCP still returns `approval_required` +- AIRI renderer intercepts `computer_use` pending actions +- Electron main shows a native approval dialog +- AIRI automatically follows up with approve/reject tool calls +- session-scoped approval reuse is limited to terminal and app open/focus actions + +That keeps approval as a user action, not a model action. + +### 5. The old remote Linux path still compiles and tests + +The previous `linux-x11` backend remains available as a legacy experimental path. +It is not the primary v1 story anymore, but it was intentionally kept compiling so existing remote smoke tooling still works. + +## Current Boundary + +Main v1 story: + +- executor: `macos-local` +- apps explicitly supported for open/focus by default: + - `Terminal` + - `Cursor` + - `Google Chrome` +- safety boundary: + - native approval dialogs + - `denyApps` + - trace / audit + - screenshot persistence + - operation budgets + +Explicit non-goals of this pass: + +- PTY/TUI terminal automation +- deep accessibility tree grounding +- strict app-level UI sandboxing +- remote sandbox hosting for other users +- Windows / Wayland / multi-monitor support + +## Commands Verified In This Checkout + +- `pnpm -F @proj-airi/computer-use-mcp typecheck` +- `pnpm -F @proj-airi/computer-use-mcp test` +- `pnpm -F @proj-airi/stage-ui typecheck` +- `pnpm -F @proj-airi/stage-tamagotchi typecheck` + +## Practical Interpretation + +The feature is now credible as: + +- a macOS desktop orchestration layer for AIRI +- a way to connect chat, MCP, terminal execution, and UI observation into one task flow +- a safer incremental path than trying to solve generic pure-vision computer use first + +It should not be pitched as: + +- a general desktop sandbox platform +- a no-approval autonomous agent +- a production-grade app-isolated desktop security boundary diff --git a/services/computer-use-mcp/README.md b/services/computer-use-mcp/README.md new file mode 100644 index 000000000..8cbb675a6 --- /dev/null +++ b/services/computer-use-mcp/README.md @@ -0,0 +1,347 @@ +# computer-use-mcp + +AIRI-specific macOS desktop orchestration MCP service. + +## Why This Exists + +This package exists because AIRI already has many useful pieces in the monorepo — +providers, chat UX, MCP attachment, desktop app surfaces, browser integrations, +tool bridges, and workflow-related logic — but those pieces are still too easy to +use as isolated features instead of one coherent agent system. + +`computer-use-mcp` is the missing execution substrate for that gap. + +The current goal is not to add "another computer use demo". The goal is to give +AIRI a unified way to: + +- observe the current desktop or browser state +- choose the right execution surface for the task +- run deterministic actions through tools and terminal commands +- keep approvals, trace, and audit artifacts attached to the run +- compose those actions into repeatable workflows instead of one-off demos + +In short: + +- AIRI remains the control plane and agent shell +- `computer-use-mcp` is the local execution and workflow substrate +- the value is in orchestration, not in cursor movement by itself + +## What It Is + +This package is no longer positioned as a generic remote computer-use experiment. +The current v1 shape is: + +- AIRI keeps the control plane: + - MCP tool surface + - approval queue protocol + - audit log + - trace history + - screenshot persistence +- `computer-use-mcp` provides a local macOS execution layer: + - window observation + - screenshots + - app open/focus + - mouse/keyboard injection + - background terminal command execution +- AIRI desktop adds a native approval adapter: + - `approval_required` still comes from MCP + - Electron shows a native dialog + - AIRI automatically calls approve/reject on the user's behalf + +The intended story is: + +- AIRI uses tools first +- visual observation is supplementary, not the primary execution path +- terminal commands are executed by a background shell runner, not by scripting Terminal tabs +- desktop/Electron/native apps and browser DOM are treated as different execution surfaces + +## Why It Is Not "Just A Mouse Toy" + +This package should not be understood as a coordinate-replay automation toy. + +What makes it different: + +- it exposes an MCP tool surface instead of a one-off macro recorder +- it keeps action policy, approval, trace history, and audit output per run +- it distinguishes between desktop control and browser DOM control instead of forcing everything through blind clicks +- it prefers deterministic execution paths (`terminal_exec`, workflows, `browser_dom_*`) before raw coordinate actions +- it is designed to be called by AIRI automatically as part of a task flow, not merely driven by a human demo operator + +That means the package is useful only when it helps AIRI turn scattered local +capabilities into one observable, controllable task system. + +## Current Executor Modes + +- `dry-run` + - default + - never injects input + - still captures best-effort local screenshots for debugging +- `macos-local` + - current primary backend + - window observation via `NSWorkspace + CGWindowList` + - input injection via Swift + Quartz `CGEvent` + - app open/focus via `open -a` and `activate` +- `linux-x11` + - retained as a legacy experimental backend + - not the main v1 story anymore + +## Tool Surface + +Desktop observation and control: + +- `desktop_get_capabilities` +- `desktop_observe_windows` +- `desktop_screenshot` +- `desktop_open_app` +- `desktop_focus_app` +- `desktop_click` +- `desktop_type_text` +- `desktop_press_keys` +- `desktop_scroll` +- `desktop_wait` + +Terminal orchestration: + +- `terminal_exec` +- `terminal_get_state` +- `terminal_reset_state` + +Clipboard bridge: + +- `secret_read_env_value` +- `clipboard_read_text` +- `clipboard_write_text` + +Browser DOM bridge: + +- `browser_agent_get_status` +- `browser_agent_run` +- `browser_dom_get_bridge_status` +- `browser_dom_get_active_tab` +- `browser_dom_read_page` +- `browser_dom_find_elements` +- `browser_dom_click` +- `browser_dom_read_input_value` +- `browser_dom_set_input_value` +- `browser_dom_check_checkbox` +- `browser_dom_select_option` +- `browser_dom_wait_for_element` +- `browser_dom_get_element_attributes` +- `browser_dom_get_computed_styles` +- `browser_dom_trigger_event` + +Approval and audit helpers: + +- `desktop_list_pending_actions` +- `desktop_approve_pending_action` +- `desktop_reject_pending_action` +- `desktop_get_session_trace` + +Workflow orchestration: + +- `workflow_open_workspace` + - reveals a workspace in Finder and opens it in the configured IDE +- `workflow_validate_workspace` + - opens the workspace, confirms `pwd`, inspects local changes, and runs a validation command such as `pnpm typecheck` +- `workflow_run_tests` + - runs a test command from the workspace root +- `workflow_inspect_failure` + - focuses the IDE and re-runs or inspects a failing command path +- `workflow_browse_and_act` + - generic browse-and-act flow for app observation and follow-up actions +- `workflow_resume` + - resumes a workflow that paused on `approval_required` + +## Policy Model + +The current macOS v1 boundary is intentionally narrow and explicit: + +- global screen coordinates are allowed for UI actions +- `allowApps` is not used as a hard gate for click/type/scroll +- `denyApps` still blocks sensitive foreground apps +- `COMPUTER_USE_OPENABLE_APPS` only gates `desktop_open_app` and `desktop_focus_app` +- AIRI itself is in the default deny list to avoid self-operation +- terminal commands always require approval +- app open/focus always require approval +- click/type/press/scroll still use per-action approval + +## Environment Variables + +Core: + +- `COMPUTER_USE_EXECUTOR` + - `dry-run`, `macos-local`, or `linux-x11` +- `COMPUTER_USE_APPROVAL_MODE` + - `actions` (default), `all`, `never` +- `COMPUTER_USE_SESSION_ROOT` + - local output directory for screenshots and `audit.jsonl` +- `COMPUTER_USE_TIMEOUT_MS` +- `COMPUTER_USE_DEFAULT_CAPTURE_AFTER` +- `COMPUTER_USE_MAX_OPERATIONS` +- `COMPUTER_USE_MAX_OPERATION_UNITS` +- `COMPUTER_USE_MAX_PENDING_ACTIONS` + +macOS orchestration: + +- `COMPUTER_USE_OPENABLE_APPS` + - default `Terminal,Cursor,Google Chrome` +- `COMPUTER_USE_DENY_APPS` + - default includes `1Password`, `Keychain`, `System Settings`, `Activity Monitor`, `AIRI` +- `COMPUTER_USE_DENY_WINDOW_TITLES` +- `COMPUTER_USE_TERMINAL_SHELL` + - default current shell, otherwise `/bin/zsh` +- `COMPUTER_USE_ALLOWED_BOUNDS` + - optional global coordinate clamp + +Browser DOM bridge: + +- `COMPUTER_USE_BROWSER_DOM_BRIDGE_ENABLED` + - default `true` +- `COMPUTER_USE_BROWSER_DOM_BRIDGE_HOST` + - default `127.0.0.1` +- `COMPUTER_USE_BROWSER_DOM_BRIDGE_PORT` + - default `8765` +- `COMPUTER_USE_BROWSER_DOM_BRIDGE_TIMEOUT_MS` + - default `10000` + +Autonomous browser agent: + +- `COMPUTER_USE_BROWSER_AGENT_ROOT` + - optional override for the embedded browser-agent workspace under `src/bin/computer_use` +- `COMPUTER_USE_PYTHON` + - optional python executable override for `browser_agent_run`; defaults to the embedded `.venv/bin/python` when present, otherwise `python3` + +Legacy remote runner: + +- `COMPUTER_USE_REMOTE_SSH_HOST` +- `COMPUTER_USE_REMOTE_SSH_USER` +- `COMPUTER_USE_REMOTE_SSH_PORT` +- `COMPUTER_USE_REMOTE_RUNNER_COMMAND` +- `COMPUTER_USE_REMOTE_DISPLAY_SIZE` +- `COMPUTER_USE_REMOTE_OBSERVATION_BASE_URL` +- `COMPUTER_USE_REMOTE_OBSERVATION_SERVE_PORT` +- `COMPUTER_USE_REMOTE_OBSERVATION_TOKEN` + +Binary overrides: + +- `COMPUTER_USE_SWIFT_BINARY` +- `COMPUTER_USE_OSASCRIPT_BINARY` +- `COMPUTER_USE_SCREENSHOT_BINARY` +- `COMPUTER_USE_OPEN_BINARY` +- `COMPUTER_USE_SSH_BINARY` +- `COMPUTER_USE_TAR_BINARY` + +## AIRI Integration + +AIRI still connects through `mcp.json`. +Example local macOS entry: + +```json +{ + "mcpServers": { + "computer_use": { + "command": "pnpm", + "args": [ + "-F", + "@proj-airi/computer-use-mcp", + "start" + ], + "cwd": "/path/to/your/airi/repo", + "env": { + "COMPUTER_USE_EXECUTOR": "macos-local", + "COMPUTER_USE_APPROVAL_MODE": "actions", + "COMPUTER_USE_OPENABLE_APPS": "Terminal,Cursor,Google Chrome" + } + } + } +} +``` + +On the AIRI desktop side, approvals are handled like this: + +1. model calls a `computer_use::*` tool +2. MCP returns `approval_required` +3. Electron shows a native approval dialog +4. AIRI automatically calls `desktop_approve_pending_action` or `desktop_reject_pending_action` +5. terminal/app approvals can be reused for the current run only + +For browser DOM automation, `computer-use-mcp` also exposes a local WebSocket bridge that matches the user's Chrome extension bridge pattern: + +1. `computer-use-mcp` listens on `ws://127.0.0.1:8765` by default +2. the unpacked browser extension connects from its offscreen document +3. AIRI can then call `browser_dom_*` MCP tools against the active browser tab + +Use the two surfaces differently: + +- `desktop_*` for AIRI itself, native macOS apps, Electron windows, Finder, Terminal, VS Code +- `browser_dom_*` for real browser pages, cross-frame DOM reads, form filling, selector-based interaction, and iframe-heavy flows +- `browser_agent_run` for goal-driven browser tasks where AIRI should delegate the web exploration loop instead of manually hard-coding each browser step + +## Validation Commands + +- `pnpm -F @proj-airi/computer-use-mcp typecheck` +- `pnpm -F @proj-airi/computer-use-mcp test` +- `pnpm -F @proj-airi/computer-use-mcp smoke:stdio` +- `pnpm -F @proj-airi/computer-use-mcp smoke:macos` +- `pnpm -F @proj-airi/computer-use-mcp e2e:airi-chat` +- `pnpm -F @proj-airi/computer-use-mcp e2e:airi-discord` + +Legacy remote validation remains available: + +- `pnpm -F @proj-airi/computer-use-mcp bootstrap:remote` +- `pnpm -F @proj-airi/computer-use-mcp smoke:remote` + +## Demo Story To Record + +If you want to record a convincing demo, show the system as an orchestrated task +runner instead of a flashy cursor dance. + +Recommended recording structure: + +1. Show the AIRI desktop window, a terminal, and the generated report directory. +2. Start the local AIRI desktop app and the `computer-use-mcp` service. +3. Show that AIRI can call the MCP tools automatically instead of only listing them. +4. Demonstrate one short task that exercises the full loop: + - observe state + - execute a tool or workflow + - produce a visible result + - persist trace / audit / screenshots +5. End by opening the generated `report.json`, `audit.jsonl`, or screenshots so the demo finishes with evidence rather than just screen motion. + +Good first demos: + +- open a workspace, confirm `pwd`, inspect local changes, and run `pnpm typecheck` +- create and run a Python hello-world project through `terminal_exec` +- use desktop control for AIRI or native apps and use `browser_dom_*` only when the task truly moves into a browser page + +### Discord integration demo + +For a management-readable AIRI demo, the Discord settings flow is more representative than a generic hello-world reply: + +1. start AIRI desktop and `services/discord-bot` +2. open `/settings/modules/messaging-discord` +3. enable the module and save settings +4. verify that the Discord bot receives the forwarded config from AIRI and reconnects itself +5. finish by opening `report.json`, screenshots, audit log, and `discord-bot.log` + +Notes: + +- for a pure local-secret run, set `AIRI_E2E_DISCORD_TOKEN` +- for a more agentic run, set `AIRI_E2E_DISCORD_TOKEN_SOURCE=portal` or `auto` and let AIRI retrieve the token from the live browser / Discord Developer Portal session +- `clipboard_read_text` / `clipboard_write_text` are the intended bridge when AIRI must move a copied token from the browser back into AIRI settings +- the observable harness keeps the token out of the desktop audit trail by applying the secret through the renderer instead of typing it through Quartz key events +- if you only want to validate the AIRI → Discord bot configuration plumbing without a real token, set `AIRI_E2E_DISCORD_ALLOW_LOGIN_FAILURE=true` + +Less convincing demos: + +- long videos of coordinate clicking with no trace output +- browser form-filling done only by screen coordinates when DOM tools were available +- tasks that cannot explain afterwards what the agent observed, executed, or verified + +## Known Limits + +- macOS only for the main v1 path +- no accessibility tree grounding yet +- PTY/TUI terminal support is product-supported on the self-acquire mainline; legacy outward terminal reroute remains secondary +- no multi-monitor orchestration policy yet +- global coordinates are allowed, so the safety boundary is approval + audit, not strict app isolation diff --git a/services/computer-use-mcp/agent.md b/services/computer-use-mcp/agent.md new file mode 100644 index 000000000..30ae7d9c2 --- /dev/null +++ b/services/computer-use-mcp/agent.md @@ -0,0 +1,218 @@ +# computer-use-mcp Agent Notes + +Scope: `services/computer-use-mcp/**` + +## Mission + +`computer-use-mcp` is AIRI's deterministic execution substrate. + +- AIRI owns planning, chat UX, approval UX, provider integration, and MCP attachment. +- `computer-use-mcp` owns execution primitives, workflow orchestration, terminal/browser/desktop surfaces, trace, audit, and safety checks. +- Treat terminal, browser, editor, and desktop operations as one task system. Do not split them into disconnected demos. + +## Current Status Snapshot + +Updated for the current terminal-lane-v2 workstream. + +The important truth is: + +- `exec` is already a real mainline surface. +- `PTY` is no longer just a loose tool set; the workflow engine now has self-acquire support. +- The service-layer terminal E2Es are green. +- The AIRI chat terminal demo is now aligned with terminal lane v2 and no longer pre-creates PTY. +- The desktop shell now distinguishes `pty_session` from `terminal_and_apps`. +- AIRI chat self-acquire is now part of the strict release gate set, so PTY mainline support is no longer intentionally held back. + +Do not rely on compressed chat summaries to resume this work. Use this file as the handoff source of truth and update it when terminal-lane behavior changes materially. + +## Terminal Lane v2: What Is Already Landed + +### 1. Terminal surface model exists + +Terminal-capable workflow steps now have explicit terminal semantics instead of pure guesswork: + +- `mode: 'exec' | 'auto' | 'pty'` +- `interaction: 'one_shot' | 'persistent'` + +The main implementation lives in: + +- `src/workflows/types.ts` +- `src/workflows/surface-resolver.ts` +- `src/terminal/interactive-patterns.ts` + +### 2. Auto surface resolution is fixed to a small rule set + +`auto` is intentionally narrow. It only upgrades to PTY when one of these is true: + +1. The current `taskId + stepId` already has a bound PTY session. +2. The step explicitly declares `interaction: 'persistent'`. +3. The command matches `KNOWN_INTERACTIVE_COMMAND_PATTERNS`. +4. A failed/timed-out exec attempt surfaces one of `INTERACTIVE_OUTPUT_MARKERS`. + +This rule set is covered by: + +- `src/workflows/surface-resolver.test.ts` +- `src/terminal/interactive-patterns.test.ts` + +### 3. Workflow engine can self-acquire PTY + +The engine already contains the v2 shape: + +- `AcquirePtyForStep` +- `StepTerminalProgress` +- suspension point `before_pty_acquire` +- PTY step family support: + - `pty_send_input` + - `pty_read_screen` + - `pty_wait_for_output` + - `pty_destroy_session` + +The main implementation lives in: + +- `src/workflows/engine.ts` + +The intended behavior is: + +- workflow resolves the terminal surface +- if PTY is needed, workflow acquires/binds PTY itself +- workflow continues inside the same workflow +- outward terminal reroute is now secondary, not the mainline proof + +### 4. Service-layer PTY self-acquire E2E exists and is green + +The current real terminal E2E for v2 is: + +- `src/bin/e2e-terminal-self-acquire.ts` + +This script now proves: + +- **no pre-created PTY** +- workflow detects an interactive command +- engine self-acquires PTY +- command executes on PTY +- step succeeds without outward reroute +- run-state / binding / audit stay consistent + +It currently uses: + +- `workflow_validate_workspace` +- an interactive `checkCommand` of `vim --version` + +This is the current service-level proof for terminal lane v2. + +### 5. AIRI chat self-acquire demo is now on the v2 path + +`src/bin/e2e-airi-chat-terminal-self-acquire.ts` follows the same product story: + +- no harness-side `pty_create` +- AIRI calls the real workflow +- the workflow self-acquires PTY for the interactive validation step +- AIRI finishes with a natural-language summary for demo use + +The latest successful reports live under: + +- `.computer-use-mcp/reports/airi-chat-terminal-self-acquire-*` + +The current package commands are: + +- `pnpm -F @proj-airi/computer-use-mcp e2e:airi-chat-terminal-self-acquire` +- `pnpm -F @proj-airi/computer-use-mcp demo:terminal-self-acquire` + +### 6. Support matrix already reflects the new direction + +Relevant entries in `src/support-matrix.ts`: + +- `terminal_exec` → `product-supported` +- `terminal_pty` → `product-supported` +- `terminal_exec_to_pty_reroute` → `covered` and explicitly labeled legacy fallback +- `terminal_auto_surface_resolution` → `covered` +- `terminal_pty_self_acquire` → `product-supported` +- `terminal_pty_step_family` → `covered` + +The current strict release gates are: + +- `pnpm -F @proj-airi/computer-use-mcp e2e:developer-workflow` +- `pnpm -F @proj-airi/computer-use-mcp e2e:terminal-exec` +- `pnpm -F @proj-airi/computer-use-mcp e2e:terminal-pty` +- `pnpm -F @proj-airi/computer-use-mcp e2e:terminal-self-acquire` +- `pnpm -F @proj-airi/computer-use-mcp e2e:airi-chat-terminal-self-acquire` + +## What Is Still Not Finished + +These are the real gaps. Do not talk yourself into thinking terminal lane is fully shipped before they are closed. + +### 1. Desktop approval semantics are improved, but still need one more explicit review + +`apps/stage-tamagotchi/src/renderer/App.vue` now distinguishes: + +- `terminal_and_apps` +- `pty_session` + +and it no longer pretends a PTY approval is the same thing as a generic terminal/app grant. + +The current intended behavior is: + +- `terminal_exec` / `open_app` / `focus_app` keep the old session-scoped auto-approve behavior +- `pty_create` stores a `pty_session` grant scope +- `pty_create` does **not** auto-approve future PTY creation requests + +This is much closer to the product model, but it is still worth reviewing whenever approval UX changes again. + +## Where To Look First + +If you are continuing terminal lane work, read these first: + +1. `src/workflows/engine.ts` +2. `src/workflows/surface-resolver.ts` +3. `src/terminal/interactive-patterns.ts` +4. `src/bin/e2e-terminal-self-acquire.ts` +5. `src/bin/e2e-airi-chat-terminal-self-acquire.ts` +6. `src/support-matrix.ts` +7. `apps/stage-tamagotchi/src/renderer/App.vue` +8. `apps/stage-tamagotchi/src/renderer/modules/computer-use-approval.ts` + +That set is enough to reconstruct the current terminal-lane-v2 state without rereading the entire repo. + +## Validation Commands + +Use these as the baseline checks for terminal lane work: + +### Service-level terminal lane + +- `pnpm -F @proj-airi/computer-use-mcp e2e:terminal-exec` +- `pnpm -F @proj-airi/computer-use-mcp e2e:terminal-pty` +- `pnpm -F @proj-airi/computer-use-mcp e2e:terminal-self-acquire` +- `pnpm -F @proj-airi/computer-use-mcp e2e:airi-chat-terminal-self-acquire` + +### Core test coverage + +- `pnpm -F @proj-airi/computer-use-mcp exec vitest run --config ./vitest.config.ts` + +### Typecheck + +- `pnpm -F @proj-airi/computer-use-mcp typecheck` +- `pnpm -F @proj-airi/stage-ui typecheck` + +If `pnpm -F @proj-airi/stage-tamagotchi typecheck` behaves oddly in the current environment, run the two underlying commands directly: + +- `pnpm -F @proj-airi/stage-tamagotchi run typecheck:node` +- `pnpm -F @proj-airi/stage-tamagotchi run typecheck:web` + +## Handoff Rules + +If you change terminal lane behavior, update this file before stopping. + +At minimum, always rewrite these four facts: + +1. Is PTY self-acquire the mainline, or does any path still depend on pre-created PTY? +2. Is AIRI chat E2E aligned with the service-level terminal lane, or still on an older path? +3. Is desktop approval using real `pty_session` semantics, or still old `terminal_and_apps` semantics? +4. Which terminal capabilities are `product-supported` vs only `covered` in `src/support-matrix.ts`? + +If those four facts are stale, the next agent will lose time re-deriving context from code. + +## Boundary Reminder + +- Keep provider-specific behavior in AIRI / `packages/stage-ui/**`. +- Keep OS-executor and workflow orchestration logic here. +- Do not expand this workstream into browser, native click/type/press, or VS Code productization until terminal lane is actually closed. diff --git a/services/computer-use-mcp/fixtures/fake-runner.mjs b/services/computer-use-mcp/fixtures/fake-runner.mjs new file mode 100644 index 000000000..ac2192b50 --- /dev/null +++ b/services/computer-use-mcp/fixtures/fake-runner.mjs @@ -0,0 +1,188 @@ +import { createInterface } from 'node:readline' + +// TODO(@nekomeowww): try now to directly embed binary / base64, even tests. `xz` warned us. +const tinyPngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Wn8vO0AAAAASUVORK5CYII=' + +const state = { + sessionTag: process.env.FAKE_RUNNER_SESSION_TAG || 'vm-local-1', + displayId: process.env.FAKE_RUNNER_DISPLAY_ID || ':99', + hostName: process.env.FAKE_RUNNER_HOST_NAME || 'fake-remote', + remoteUser: process.env.FAKE_RUNNER_REMOTE_USER || 'airi', + width: Number.parseInt(process.env.FAKE_RUNNER_WIDTH || '1280', 10), + height: Number.parseInt(process.env.FAKE_RUNNER_HEIGHT || '720', 10), + observationBaseUrl: process.env.FAKE_RUNNER_OBSERVATION_BASE_URL || '', +} + +function executionTarget() { + return { + mode: 'remote', + transport: 'ssh-stdio', + hostName: state.hostName, + remoteUser: state.remoteUser, + displayId: state.displayId, + sessionTag: state.sessionTag, + isolated: true, + tainted: false, + } +} + +function permissionInfo() { + return { + screenRecording: { + status: 'granted', + target: `${state.displayId} via scrot`, + checkedBy: 'scrot', + }, + accessibility: { + status: 'unsupported', + target: `${state.displayId} linux-x11 session`, + note: 'linux-x11 runner does not rely on accessibility APIs', + }, + automationToSystemEvents: { + status: 'unsupported', + target: `${state.displayId} linux-x11 session`, + note: 'linux-x11 runner does not use System Events', + }, + } +} + +function displayInfo() { + return { + available: true, + platform: 'linux', + logicalWidth: state.width, + logicalHeight: state.height, + pixelWidth: state.width, + pixelHeight: state.height, + scaleFactor: 1, + isRetina: false, + note: `managed virtual X session ${state.displayId}`, + } +} + +function writeResponse(response) { + process.stdout.write(`${JSON.stringify(response)}\n`) +} + +const rl = createInterface({ + input: process.stdin, + crlfDelay: Infinity, +}) + +rl.on('line', (line) => { + const trimmed = line.trim() + if (!trimmed) { + return + } + + const request = JSON.parse(trimmed) + if (process.env.FAKE_RUNNER_CLOSE_ON_MUTATION === '1' && ['click', 'typeText', 'pressKeys', 'scroll'].includes(request.method)) { + process.exit(1) + } + + switch (request.method) { + case 'initialize': + writeResponse({ + id: request.id, + ok: true, + result: { + executionTarget: executionTarget(), + displayInfo: displayInfo(), + permissionInfo: permissionInfo(), + }, + }) + return + case 'getExecutionTarget': + writeResponse({ + id: request.id, + ok: true, + result: executionTarget(), + }) + return + case 'getDisplayInfo': + writeResponse({ + id: request.id, + ok: true, + result: displayInfo(), + }) + return + case 'getForegroundContext': + writeResponse({ + id: request.id, + ok: true, + result: { + available: true, + appName: 'mousepad', + windowTitle: 'Mousepad', + platform: 'linux', + }, + }) + return + case 'getPermissionInfo': + writeResponse({ + id: request.id, + ok: true, + result: permissionInfo(), + }) + return + case 'takeScreenshot': + writeResponse({ + id: request.id, + ok: true, + result: { + dataBase64: tinyPngBase64, + mimeType: 'image/png', + ...(state.observationBaseUrl + ? { + publicUrl: `${state.observationBaseUrl.replace(/\/$/, '')}/fake-screenshot.png`, + } + : {}), + width: state.width, + height: state.height, + executionTarget: executionTarget(), + }, + }) + return + case 'click': + case 'typeText': + case 'pressKeys': + case 'scroll': + case 'wait': + writeResponse({ + id: request.id, + ok: true, + result: { + performed: true, + backend: 'linux-x11', + notes: [`${request.method} executed`], + executionTarget: executionTarget(), + }, + }) + return + case 'openTestTarget': + writeResponse({ + id: request.id, + ok: true, + result: { + launched: true, + appName: 'mousepad', + windowTitle: 'Mousepad', + recommendedClickPoint: { + x: 180, + y: 150, + }, + executionTarget: executionTarget(), + }, + }) + return + case 'shutdown': + writeResponse({ + id: request.id, + ok: true, + result: { + ok: true, + }, + }) + process.exit(0) + } +}) diff --git a/services/computer-use-mcp/fixtures/interactive-echo.mjs b/services/computer-use-mcp/fixtures/interactive-echo.mjs new file mode 100644 index 000000000..db9c814d4 --- /dev/null +++ b/services/computer-use-mcp/fixtures/interactive-echo.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +/** + * Deterministic interactive program for PTY E2E testing. + * + * Behaviour: + * 1. Prints "READY> " prompt to stdout + * 2. Reads one line of input from stdin + * 3. Prints "ECHO: " followed by "DONE" + * 4. Exits with code 0 + * + * This avoids unpredictable TUI programs (vim, less, top) while still + * exercising the real PTY read/write/lifecycle path. + */ + +import { createInterface } from 'node:readline' + +const rl = createInterface({ input: process.stdin, output: process.stdout }) + +process.stdout.write('READY> ') + +rl.once('line', (line) => { + process.stdout.write(`ECHO: ${line}\n`) + process.stdout.write('DONE\n') + rl.close() + process.exit(0) +}) diff --git a/services/computer-use-mcp/fixtures/text-target.html b/services/computer-use-mcp/fixtures/text-target.html new file mode 100644 index 000000000..c073dd43c --- /dev/null +++ b/services/computer-use-mcp/fixtures/text-target.html @@ -0,0 +1,61 @@ + + + + + + AIRI Computer Use Text Target + + + +
+

AIRI Text Target

+

Use this page as the low-noise fallback target when TextEdit is unavailable inside the VM.

+ +
+ + diff --git a/services/computer-use-mcp/package.json b/services/computer-use-mcp/package.json new file mode 100644 index 000000000..1af673bfc --- /dev/null +++ b/services/computer-use-mcp/package.json @@ -0,0 +1,57 @@ +{ + "name": "@proj-airi/computer-use-mcp", + "type": "module", + "version": "0.1.0", + "private": true, + "description": "AIRI-specific macOS desktop orchestration MCP service", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "bin": "./dist/bin/run.mjs", + "files": [ + "README.md", + "dist", + "fixtures", + "package.json" + ], + "scripts": { + "dev": "tsx ./src/bin/run.ts", + "start": "tsx ./src/bin/run.ts", + "e2e:airi-chat": "tsx ./src/bin/e2e-airi-chat-observable.ts", + "e2e:airi-chat-terminal-self-acquire": "tsx ./src/bin/e2e-airi-chat-terminal-self-acquire.ts", + "demo:terminal-self-acquire": "tsx ./src/bin/e2e-airi-chat-terminal-self-acquire.ts", + "e2e:airi-discord": "tsx ./src/bin/e2e-airi-discord-observable.ts", + "e2e:airi-discord-agentic": "tsx ./src/bin/e2e-airi-discord-agentic.ts", + "e2e:developer-workflow": "tsx ./src/bin/e2e-developer-workflow.ts", + "e2e:browser-reroute": "tsx ./src/bin/e2e-browser-reroute.ts", + "e2e:terminal-exec": "tsx ./src/bin/e2e-terminal-exec.ts", + "e2e:terminal-pty": "tsx ./src/bin/e2e-terminal-pty.ts", + "e2e:terminal-self-acquire": "tsx ./src/bin/e2e-terminal-self-acquire.ts", + "bootstrap:remote": "tsx ./src/bin/bootstrap-remote.ts", + "smoke:remote": "tsx ./src/bin/smoke-remote.ts", + "smoke:stdio": "tsx ./src/bin/smoke-stdio.ts", + "smoke:macos": "tsx ./src/bin/smoke-macos.ts", + "smoke:workflow": "tsx ./src/bin/smoke-workflow.ts", + "mcp:inspector": "pnpx @modelcontextprotocol/inspector pnpm -F @proj-airi/computer-use-mcp start", + "build": "tsdown", + "typecheck": "tsc --noEmit", + "test": "vitest run --config ./vitest.config.ts" + }, + "dependencies": { + "@modelcontextprotocol/sdk": "catalog:", + "node-pty": "catalog:", + "ws": "^8.19.0", + "zod": "^4.3.6" + } +} diff --git a/services/computer-use-mcp/src/accessibility/ax-tree.test.ts b/services/computer-use-mcp/src/accessibility/ax-tree.test.ts new file mode 100644 index 000000000..3e489a30a --- /dev/null +++ b/services/computer-use-mcp/src/accessibility/ax-tree.test.ts @@ -0,0 +1,143 @@ +import type { AXNode, AXSnapshot } from '../accessibility/types' + +import { describe, expect, it } from 'vitest' + +import { findAXNodeByUid, formatAXSnapshotAsText } from '../accessibility/ax-tree' + +function createTestSnapshot(overrides: Partial = {}): AXSnapshot { + const root: AXNode = { + uid: '1_0', + role: 'AXApplication', + title: 'TestApp', + children: [ + { + uid: '1_1', + role: 'AXWindow', + title: 'Main Window', + children: [ + { + uid: '1_2', + role: 'AXButton', + title: 'OK', + bounds: { x: 100, y: 200, width: 80, height: 30 }, + children: [], + }, + { + uid: '1_3', + role: 'AXTextField', + title: 'Name', + value: 'Hello World', + focused: true, + children: [], + }, + { + uid: '1_4', + role: 'AXButton', + title: 'Cancel', + enabled: false, + children: [], + }, + ], + }, + ], + } + + const uidToNode = new Map() + function index(node: AXNode) { + uidToNode.set(node.uid, node) + for (const child of node.children) { + index(child) + } + } + index(root) + + return { + snapshotId: '1', + pid: 1234, + appName: 'TestApp', + root, + uidToNode, + capturedAt: '2025-01-01T00:00:00.000Z', + maxDepth: 15, + truncated: false, + ...overrides, + } +} + +describe('formatAXSnapshotAsText', () => { + it('formats a basic tree with uids', () => { + const snapshot = createTestSnapshot() + const text = formatAXSnapshotAsText(snapshot) + + expect(text).toContain('[AXTree] TestApp (pid 1234)') + expect(text).toContain('[1_0] AXApplication "TestApp"') + expect(text).toContain('[1_2] AXButton "OK"') + expect(text).toContain('[1_3] AXTextField "Name" val="Hello World" [focused]') + expect(text).toContain('[1_4] AXButton "Cancel" [disabled]') + }) + + it('includes bounds when requested', () => { + const snapshot = createTestSnapshot() + const text = formatAXSnapshotAsText(snapshot, { includeBounds: true }) + + expect(text).toContain('@(100,200 80x30)') + }) + + it('omits uids when requested', () => { + const snapshot = createTestSnapshot() + const text = formatAXSnapshotAsText(snapshot, { includeUids: false }) + + expect(text).not.toContain('[1_0]') + expect(text).toContain('AXApplication "TestApp"') + }) + + it('marks truncated snapshots', () => { + const snapshot = createTestSnapshot({ truncated: true }) + const text = formatAXSnapshotAsText(snapshot) + + expect(text).toContain('[TRUNCATED]') + }) + + it('truncates long values', () => { + const root: AXNode = { + uid: '1_0', + role: 'AXStaticText', + value: 'A'.repeat(200), + children: [], + } + const uidToNode = new Map([['1_0', root]]) + const snapshot: AXSnapshot = { + snapshotId: '1', + pid: 1, + appName: 'Test', + root, + uidToNode, + capturedAt: '2025-01-01T00:00:00.000Z', + maxDepth: 15, + truncated: false, + } + const text = formatAXSnapshotAsText(snapshot) + + expect(text).toContain('...') + // Value should be truncated to 80 chars + expect(text).not.toContain('A'.repeat(200)) + }) +}) + +describe('findAXNodeByUid', () => { + it('finds a node by uid', () => { + const snapshot = createTestSnapshot() + const node = findAXNodeByUid(snapshot, '1_2') + + expect(node).toBeDefined() + expect(node!.role).toBe('AXButton') + expect(node!.title).toBe('OK') + }) + + it('returns undefined for non-existent uid', () => { + const snapshot = createTestSnapshot() + const node = findAXNodeByUid(snapshot, 'nonexistent') + + expect(node).toBeUndefined() + }) +}) diff --git a/services/computer-use-mcp/src/accessibility/ax-tree.ts b/services/computer-use-mcp/src/accessibility/ax-tree.ts new file mode 100644 index 000000000..5427df13f --- /dev/null +++ b/services/computer-use-mcp/src/accessibility/ax-tree.ts @@ -0,0 +1,342 @@ +/** + * macOS Accessibility tree capture via Swift + AXUIElement API. + * + * Runs an inline Swift script that walks the AXTree of the frontmost (or + * specified) application and returns a JSON tree. The tree is then parsed + * into the `AXSnapshot` structure used by the MCP tool layer. + */ + +import type { ComputerUseConfig } from '../types' +import type { AXNode, AXSnapshot, AXSnapshotRequest, AXSnapshotTextOptions } from './types' + +import { platform } from 'node:process' + +import { runSwiftScript } from '../utils/swift' + +let nextSnapshotId = 1 + +/** + * Swift source that uses ApplicationServices / AXUIElement to walk the + * accessibility tree of a target process. Input is passed via the + * COMPUTER_USE_SWIFT_STDIN environment variable as JSON. + * + * Output format: + * ```json + * { + * "pid": 1234, + * "appName": "Finder", + * "root": { "role": "AXApplication", "title": "Finder", ... }, + * "truncated": false + * } + * ``` + */ +function axTreeScript(): string { + return String.raw` +import ApplicationServices +import AppKit +import Foundation + +struct AXNodeJSON: Encodable { + let role: String + let title: String? + let value: String? + let description: String? + let enabled: Bool? + let focused: Bool? + let bounds: BoundsJSON? + let children: [AXNodeJSON] +} + +struct BoundsJSON: Encodable { + let x: Int + let y: Int + let width: Int + let height: Int +} + +struct OutputJSON: Encodable { + let pid: Int32 + let appName: String + let root: AXNodeJSON? + let truncated: Bool +} + +func getStringAttr(_ element: AXUIElement, _ attr: String) -> String? { + var value: AnyObject? + guard AXUIElementCopyAttributeValue(element, attr as CFString, &value) == .success else { return nil } + return value as? String +} + +func getBoolAttr(_ element: AXUIElement, _ attr: String) -> Bool? { + var value: AnyObject? + guard AXUIElementCopyAttributeValue(element, attr as CFString, &value) == .success else { return nil } + if let num = value as? NSNumber { return num.boolValue } + return nil +} + +func getBounds(_ element: AXUIElement) -> BoundsJSON? { + var posValue: AnyObject? + var sizeValue: AnyObject? + guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as String as CFString, &posValue) == .success, + AXUIElementCopyAttributeValue(element, kAXSizeAttribute as String as CFString, &sizeValue) == .success + else { return nil } + + let posType = AXValueGetType(posValue as! AXValue) + let sizeType = AXValueGetType(sizeValue as! AXValue) + guard posType == .cgPoint, sizeType == .cgSize else { return nil } + + var point = CGPoint.zero + var size = CGSize.zero + AXValueGetValue(posValue as! AXValue, .cgPoint, &point) + AXValueGetValue(sizeValue as! AXValue, .cgSize, &size) + + return BoundsJSON( + x: Int(point.x.rounded()), + y: Int(point.y.rounded()), + width: Int(size.width.rounded()), + height: Int(size.height.rounded()) + ) +} + +func walkTree(_ element: AXUIElement, depth: Int, maxDepth: Int, nodeCount: inout Int, maxNodes: Int, verbose: Bool) -> AXNodeJSON? { + if depth > maxDepth || nodeCount >= maxNodes { return nil } + nodeCount += 1 + + let role = getStringAttr(element, kAXRoleAttribute as String) ?? "" + let title = getStringAttr(element, kAXTitleAttribute as String) + let valueStr: String? = { + var raw: AnyObject? + guard AXUIElementCopyAttributeValue(element, kAXValueAttribute as String as CFString, &raw) == .success else { return nil } + if let s = raw as? String { return s.count > 500 ? String(s.prefix(500)) : s } + if let n = raw as? NSNumber { return n.stringValue } + return nil + }() + let desc = getStringAttr(element, kAXDescriptionAttribute as String) + + if !verbose && role.isEmpty && title == nil && desc == nil && valueStr == nil { + return nil + } + + let enabled = getBoolAttr(element, kAXEnabledAttribute as String) + let focused = getBoolAttr(element, kAXFocusedAttribute as String) + let bounds = getBounds(element) + + var childNodes: [AXNodeJSON] = [] + var childrenRef: AnyObject? + if AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as String as CFString, &childrenRef) == .success, + let children = childrenRef as? [AXUIElement] { + for child in children { + if let childNode = walkTree(child, depth: depth + 1, maxDepth: maxDepth, nodeCount: &nodeCount, maxNodes: maxNodes, verbose: verbose) { + childNodes.append(childNode) + } + } + } + + return AXNodeJSON( + role: role, + title: title, + value: valueStr, + description: desc, + enabled: enabled, + focused: focused, + bounds: bounds, + children: childNodes + ) +} + +let environment = ProcessInfo.processInfo.environment +let rawInput = environment["COMPUTER_USE_SWIFT_STDIN"] ?? "{}" +let inputData = rawInput.data(using: .utf8) ?? Data() +let input = (try? JSONSerialization.jsonObject(with: inputData)) as? [String: Any] ?? [:] + +let maxDepth = (input["maxDepth"] as? Int) ?? 15 +let maxNodes = (input["maxNodes"] as? Int) ?? 2000 +let verbose = (input["verbose"] as? Bool) ?? false +let targetPid: Int32? = (input["pid"] as? Int).map { Int32($0) } + +let pid: Int32 +let appName: String + +if let targetPid { + pid = targetPid + let app = NSRunningApplication(processIdentifier: targetPid) + appName = app?.localizedName ?? "pid:\(targetPid)" +} else { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { + let output = OutputJSON(pid: 0, appName: "unknown", root: nil, truncated: false) + let data = try JSONEncoder().encode(output) + print(String(data: data, encoding: .utf8)!) + exit(0) + } + pid = frontApp.processIdentifier + appName = frontApp.localizedName ?? "unknown" +} + +let appElement = AXUIElementCreateApplication(pid) +var nodeCount = 0 +let root = walkTree(appElement, depth: 0, maxDepth: maxDepth, nodeCount: &nodeCount, maxNodes: maxNodes, verbose: verbose) + +let output = OutputJSON(pid: pid, appName: appName, root: root, truncated: nodeCount >= maxNodes) +let encoder = JSONEncoder() +let data = try encoder.encode(output) +print(String(data: data, encoding: .utf8)!) +` +} + +interface RawAXNode { + role: string + title?: string + value?: string + description?: string + enabled?: boolean + focused?: boolean + bounds?: { x: number, y: number, width: number, height: number } + children?: RawAXNode[] +} + +interface RawAXOutput { + pid: number + appName: string + root?: RawAXNode + truncated: boolean +} + +/** + * Assign stable uids to each node and build a flat lookup table. + */ +function assignUids( + raw: RawAXNode, + snapshotId: string, + uidToNode: Map, +): AXNode { + let counter = 0 + + function walk(node: RawAXNode): AXNode { + const uid = `${snapshotId}_${counter++}` + const axNode: AXNode = { + uid, + role: node.role, + title: node.title, + value: node.value, + description: node.description, + enabled: node.enabled, + focused: node.focused, + bounds: node.bounds, + children: (node.children ?? []).map(walk), + } + uidToNode.set(uid, axNode) + return axNode + } + + return walk(raw) +} + +/** + * Capture the accessibility tree of the frontmost (or specified) macOS app. + */ +export async function captureAXTree( + config: ComputerUseConfig, + request: AXSnapshotRequest = {}, +): Promise { + if (platform !== 'darwin') { + throw new Error('accessibility tree capture is only supported on macOS') + } + + const { stdout } = await runSwiftScript({ + swiftBinary: config.binaries.swift, + timeoutMs: config.timeoutMs, + source: axTreeScript(), + stdinPayload: { + pid: request.pid, + maxDepth: request.maxDepth ?? 15, + maxNodes: request.maxNodes ?? 2000, + verbose: request.verbose ?? false, + }, + }) + + const raw = JSON.parse(stdout.trim()) as RawAXOutput + const snapshotId = String(nextSnapshotId++) + const uidToNode = new Map() + + const root: AXNode = raw.root + ? assignUids(raw.root, snapshotId, uidToNode) + : { uid: `${snapshotId}_0`, role: 'AXApplication', children: [] } + + if (!raw.root) { + uidToNode.set(root.uid, root) + } + + return { + snapshotId, + pid: raw.pid, + appName: raw.appName, + root, + uidToNode, + capturedAt: new Date().toISOString(), + maxDepth: request.maxDepth ?? 15, + truncated: raw.truncated, + } +} + +/** + * Format an AXSnapshot as an indented text tree suitable for LLM context. + */ +export function formatAXSnapshotAsText( + snapshot: AXSnapshot, + options: AXSnapshotTextOptions = {}, +): string { + const indent = options.indent ?? ' ' + const includeBounds = options.includeBounds ?? false + const includeUids = options.includeUids ?? true + + const lines: string[] = [] + lines.push(`[AXTree] ${snapshot.appName} (pid ${snapshot.pid})${snapshot.truncated ? ' [TRUNCATED]' : ''}`) + + function walk(node: AXNode, depth: number) { + const prefix = indent.repeat(depth) + const parts: string[] = [] + + if (includeUids) { + parts.push(`[${node.uid}]`) + } + + parts.push(node.role || '(no role)') + + if (node.title) { + parts.push(`"${node.title}"`) + } + if (node.value) { + const truncated = node.value.length > 80 ? `${node.value.slice(0, 77)}...` : node.value + parts.push(`val="${truncated}"`) + } + if (node.description) { + parts.push(`desc="${node.description}"`) + } + if (node.focused) { + parts.push('[focused]') + } + if (node.enabled === false) { + parts.push('[disabled]') + } + if (includeBounds && node.bounds) { + const b = node.bounds + parts.push(`@(${b.x},${b.y} ${b.width}x${b.height})`) + } + + lines.push(`${prefix}${parts.join(' ')}`) + + for (const child of node.children) { + walk(child, depth + 1) + } + } + + walk(snapshot.root, 0) + return lines.join('\n') +} + +/** + * Find a node by uid in the snapshot. + */ +export function findAXNodeByUid(snapshot: AXSnapshot, uid: string): AXNode | undefined { + return snapshot.uidToNode.get(uid) +} diff --git a/services/computer-use-mcp/src/accessibility/index.ts b/services/computer-use-mcp/src/accessibility/index.ts new file mode 100644 index 000000000..8d81c4c79 --- /dev/null +++ b/services/computer-use-mcp/src/accessibility/index.ts @@ -0,0 +1,2 @@ +export { captureAXTree, findAXNodeByUid, formatAXSnapshotAsText } from './ax-tree' +export type { AXNode, AXSnapshot, AXSnapshotRequest, AXSnapshotTextOptions } from './types' diff --git a/services/computer-use-mcp/src/accessibility/types.ts b/services/computer-use-mcp/src/accessibility/types.ts new file mode 100644 index 000000000..407d1c101 --- /dev/null +++ b/services/computer-use-mcp/src/accessibility/types.ts @@ -0,0 +1,66 @@ +/** + * Accessibility tree types for native macOS UI grounding. + * + * Uses the macOS Accessibility API (AXUIElement) via Swift to query the + * AXTree of the focused application. This provides semantic structure + * (roles, labels, values, bounds) that complements pixel-based screenshots. + */ + +export interface AXNode { + /** Stable uid for this node within the snapshot */ + uid: string + role: string + title?: string + value?: string + description?: string + /** Whether the element can receive focus / interaction */ + enabled?: boolean + focused?: boolean + /** Screen-coordinate bounding rect */ + bounds?: { + x: number + y: number + width: number + height: number + } + children: AXNode[] +} + +export interface AXSnapshot { + /** Unique id for this snapshot (monotonically increasing) */ + snapshotId: string + /** PID of the app whose tree was captured */ + pid: number + /** Application name */ + appName: string + /** Root of the AXTree */ + root: AXNode + /** Flat lookup table: uid → node */ + uidToNode: Map + /** When the snapshot was taken */ + capturedAt: string + /** Max depth used during capture */ + maxDepth: number + /** Whether the tree was truncated due to depth/node limits */ + truncated: boolean +} + +export interface AXSnapshotRequest { + /** Target a specific PID instead of frontmost app */ + pid?: number + /** Maximum tree depth to traverse (default: 15) */ + maxDepth?: number + /** Maximum total nodes to collect (default: 2000) */ + maxNodes?: number + /** Whether to include nodes with empty roles/titles (default: false) */ + verbose?: boolean +} + +export interface AXSnapshotTextOptions { + /** Indentation string per level */ + indent?: string + /** Whether to include bounds info */ + includeBounds?: boolean + /** Whether to include uid annotations */ + includeUids?: boolean +} diff --git a/services/computer-use-mcp/src/app-aliases.test.ts b/services/computer-use-mcp/src/app-aliases.test.ts new file mode 100644 index 000000000..f5d6a69a6 --- /dev/null +++ b/services/computer-use-mcp/src/app-aliases.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' + +import { appNamesMatch, canonicalizeKnownAppName, findKnownAppMention, getKnownAppLaunchNames, normalizeConfiguredAppAction, resolveConfiguredOpenableApp } from './app-aliases' + +describe('app aliases', () => { + it('matches VS Code aliases to Visual Studio Code', () => { + expect(appNamesMatch('VS Code', 'Visual Studio Code')).toBe(true) + expect(appNamesMatch('vscode', 'Visual Studio Code')).toBe(true) + expect(appNamesMatch('Visual Studio Code for mac', 'Visual Studio Code')).toBe(true) + expect(canonicalizeKnownAppName('VS Code')).toBe('Visual Studio Code') + expect(resolveConfiguredOpenableApp('VS Code', ['Finder', 'Visual Studio Code'])).toBe('Visual Studio Code') + expect(resolveConfiguredOpenableApp('Visual Studio Code for mac', ['Finder', 'Visual Studio Code'])).toBe('Visual Studio Code') + expect(getKnownAppLaunchNames('VS Code')).toContain('Visual Studio Code for mac') + }) + + it('normalizes open_app actions to the configured canonical app name', () => { + expect(normalizeConfiguredAppAction({ + kind: 'open_app', + input: { app: 'VS Code' }, + }, ['Finder', 'Visual Studio Code'])).toEqual({ + kind: 'open_app', + input: { app: 'Visual Studio Code' }, + }) + }) + + it('finds known app mentions in workflow labels', () => { + expect(findKnownAppMention('Open project in VS Code')).toBe('Visual Studio Code') + expect(findKnownAppMention('Reveal folder in Finder')).toBe('Finder') + }) +}) diff --git a/services/computer-use-mcp/src/app-aliases.ts b/services/computer-use-mcp/src/app-aliases.ts new file mode 100644 index 000000000..3f6f76905 --- /dev/null +++ b/services/computer-use-mcp/src/app-aliases.ts @@ -0,0 +1,100 @@ +import type { ActionInvocation } from './types' + +interface KnownAppDefinition { + canonical: string + aliases: string[] + launchNames?: string[] +} + +const knownApps: KnownAppDefinition[] = [ + { canonical: 'Finder', aliases: ['finder'] }, + { canonical: 'Terminal', aliases: ['terminal', 'terminal.app'] }, + { canonical: 'Cursor', aliases: ['cursor'] }, + { + canonical: 'Visual Studio Code', + aliases: ['visual studio code', 'visual studio code for mac', 'vs code', 'vscode', 'code'], + launchNames: ['Visual Studio Code', 'Visual Studio Code for mac'], + }, + { canonical: 'Google Chrome', aliases: ['google chrome', 'chrome'] }, + { canonical: 'Electron', aliases: ['electron'] }, +] + +function normalizeAppNameKey(value: string) { + return value + .trim() + .toLowerCase() + .replace(/\.app$/u, '') + .replace(/\s+/gu, ' ') +} + +function getCanonicalKnownAppName(value: string) { + const requestedKey = normalizeAppNameKey(value) + const match = knownApps.find(app => app.aliases.some(alias => normalizeAppNameKey(alias) === requestedKey)) + return match?.canonical +} + +function getKnownAppDefinition(value: string) { + const requestedKey = normalizeAppNameKey(value) + return knownApps.find((app) => { + const candidates = [app.canonical, ...app.aliases, ...(app.launchNames ?? [])] + return candidates.some(candidate => normalizeAppNameKey(candidate) === requestedKey) + }) +} + +export function canonicalizeKnownAppName(value: string) { + return getCanonicalKnownAppName(value) ?? value.trim() +} + +export function getKnownAppLaunchNames(value: string) { + const definition = getKnownAppDefinition(value) + if (!definition) { + return [value.trim()] + } + + return Array.from(new Set([definition.canonical, ...(definition.launchNames ?? []), value.trim()])) +} + +export function appNamesMatch(left: string | undefined, right: string | undefined) { + if (!left || !right) { + return false + } + + const leftKey = normalizeAppNameKey(left) + const rightKey = normalizeAppNameKey(right) + if (leftKey === rightKey) { + return true + } + + const leftCanonical = canonicalizeKnownAppName(left) + const rightCanonical = canonicalizeKnownAppName(right) + return normalizeAppNameKey(leftCanonical) === normalizeAppNameKey(rightCanonical) +} + +export function resolveConfiguredOpenableApp(requested: string, openableApps: string[]) { + return openableApps.find(candidate => appNamesMatch(candidate, requested)) +} + +export function normalizeConfiguredAppAction(action: ActionInvocation, openableApps: string[]): ActionInvocation { + if (action.kind !== 'open_app' && action.kind !== 'focus_app') { + return action + } + + const resolvedApp = resolveConfiguredOpenableApp(action.input.app, openableApps) + if (!resolvedApp) { + return action + } + + return { + ...action, + input: { + ...action.input, + app: resolvedApp, + }, + } +} + +export function findKnownAppMention(text: string) { + const normalized = normalizeAppNameKey(text) + const match = knownApps.find(app => app.aliases.some(alias => normalized.includes(normalizeAppNameKey(alias)))) + return match?.canonical +} diff --git a/services/computer-use-mcp/src/bin/bootstrap-remote.ts b/services/computer-use-mcp/src/bin/bootstrap-remote.ts new file mode 100644 index 000000000..acb220da9 --- /dev/null +++ b/services/computer-use-mcp/src/bin/bootstrap-remote.ts @@ -0,0 +1,119 @@ +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { resolveComputerUseConfig } from '../config' +import { normalizeRemoteShellPath, runRemoteCommand, uploadDirectoryToRemote } from '../remote/ssh' +import { runProcess } from '../utils/process' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const distDir = resolve(packageDir, 'dist') +const remoteInstallDir = normalizeRemoteShellPath(env.COMPUTER_USE_REMOTE_INSTALL_DIR?.trim() || '${HOME}/.local/share/airi-desktop-runner') +const remoteRunnerPath = normalizeRemoteShellPath(env.COMPUTER_USE_REMOTE_RUNNER_COMMAND?.trim() || '${HOME}/.local/bin/airi-desktop-runner') + +async function buildLocalBundle() { + await runProcess('pnpm', ['build'], { + cwd: packageDir, + timeoutMs: 180_000, + env: process.env, + }) +} + +async function installRemoteDependencies() { + const config = resolveComputerUseConfig() + if (env.COMPUTER_USE_REMOTE_SKIP_PACKAGE_INSTALL === '1') { + return + } + + const packages = [ + 'nodejs', + 'xvfb', + 'xauth', + 'xdotool', + 'wmctrl', + 'scrot', + 'openbox', + 'x11-utils', + 'x11-xserver-utils', + 'mousepad', + 'xdg-utils', + ].join(' ') + + await runRemoteCommand(config, ` +if ! command -v apt-get >/dev/null 2>&1; then + echo "apt-get is required to bootstrap the remote runner" >&2 + exit 18 +fi + +if ! sudo -n true >/dev/null 2>&1; then + echo "passwordless sudo is required for bootstrap:remote, or rerun with COMPUTER_USE_REMOTE_SKIP_PACKAGE_INSTALL=1 after installing dependencies manually" >&2 + exit 17 +fi + +sudo env DEBIAN_FRONTEND=noninteractive apt-get update +sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y ${packages} +`, { + timeoutMs: 240_000, + }) +} + +async function installRemoteRunner() { + const config = resolveComputerUseConfig() + await runRemoteCommand(config, ` +mkdir -p ${remoteInstallDir} +mkdir -p $(dirname ${remoteRunnerPath}) +`, { + timeoutMs: 30_000, + }) + + await uploadDirectoryToRemote(config, { + sourceDir: distDir, + remoteDir: `${remoteInstallDir}/dist`, + timeoutMs: 180_000, + }) + + const wrapper = `#!/usr/bin/env sh +set -eu +exec node ${remoteInstallDir}/dist/bin/runner.mjs +` + + await runRemoteCommand(config, `cat > ${remoteRunnerPath} && chmod +x ${remoteRunnerPath}`, { + stdin: wrapper, + timeoutMs: 30_000, + }) + + await runRemoteCommand(config, ` +test -x ${remoteRunnerPath} +test -f ${remoteInstallDir}/dist/bin/runner.mjs +node --version +`, { + timeoutMs: 15_000, + }) +} + +async function main() { + const config = resolveComputerUseConfig() + if (!config.remoteSshHost || !config.remoteSshUser) { + throw new Error('bootstrap:remote requires COMPUTER_USE_REMOTE_SSH_HOST and COMPUTER_USE_REMOTE_SSH_USER') + } + + await buildLocalBundle() + await installRemoteDependencies() + await installRemoteRunner() + + console.info(JSON.stringify({ + ok: true, + remote: { + host: config.remoteSshHost, + user: config.remoteSshUser, + port: config.remoteSshPort, + installDir: remoteInstallDir, + runnerCommand: remoteRunnerPath, + }, + }, null, 2)) +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/demo-hello-world.ts b/services/computer-use-mcp/src/bin/demo-hello-world.ts new file mode 100644 index 000000000..579a333c2 --- /dev/null +++ b/services/computer-use-mcp/src/bin/demo-hello-world.ts @@ -0,0 +1,157 @@ +/** + * Demo: use computer-use-mcp's terminal_exec tool via MCP client + * to create a Python hello-world project and run it. + */ +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +async function main() { + console.info('🚀 Starting computer-use-mcp server …') + + const transport = new StdioClientTransport({ + command: 'pnpm', + args: ['start'], + cwd: packageDir, + env: { + ...env, + // Use macos-local executor so terminal_exec actually runs commands + COMPUTER_USE_EXECUTOR: 'macos-local', + // Skip manual approval for this demo + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_SESSION_TAG: 'demo-hello-world', + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,2560,1600', + }, + stderr: 'pipe', + }) + + const client = new Client({ + name: 'demo-hello-world-client', + version: '0.1.0', + }) + + // Pipe server stderr so we can see logs + transport.stderr?.on('data', (chunk: { toString: (enc: string) => string }) => { + const text = chunk.toString('utf-8').trim() + if (text) { + console.error(` [server] ${text}`) + } + }) + + try { + await client.connect(transport) + console.info('✅ Connected to computer-use-mcp server') + + // 1. List available tools + const tools = await client.listTools() + console.info(`📋 Available tools (${tools.tools.length}):`) + for (const t of tools.tools) { + console.info(` - ${t.name}`) + } + + // 2. Step 1: Create folder + Python file via terminal_exec + console.info('\n📁 Step 1: Creating folder ~/hello-python-project …') + const mkdirResult = await client.callTool({ + name: 'terminal_exec', + arguments: { + command: 'mkdir -p ~/hello-python-project', + timeoutMs: 10_000, + }, + }) + printResult('mkdir', mkdirResult) + + // 3. Step 2: Write main.py + console.info('\n📝 Step 2: Writing main.py …') + const writeResult = await client.callTool({ + name: 'terminal_exec', + arguments: { + command: `cat > ~/hello-python-project/main.py << 'PYEOF' +#!/usr/bin/env python3 +"""Hello World project — created by AIRI computer-use-mcp""" + +def main(): + print("Hello World! 🌍") + print("This project was created by AIRI computer-use-mcp terminal_exec tool.") + +if __name__ == "__main__": + main() +PYEOF`, + timeoutMs: 10_000, + }, + }) + printResult('write main.py', writeResult) + + // 4. Step 3: Run it! + console.info('\n🐍 Step 3: Running python3 ~/hello-python-project/main.py …') + const runResult = await client.callTool({ + name: 'terminal_exec', + arguments: { + command: 'python3 ~/hello-python-project/main.py', + cwd: `${env.HOME}/hello-python-project`, + timeoutMs: 15_000, + }, + }) + printResult('run main.py', runResult) + + // 5. Step 4: Show the project structure + console.info('\n📂 Step 4: Listing project contents …') + const lsResult = await client.callTool({ + name: 'terminal_exec', + arguments: { + command: 'ls -la ~/hello-python-project && echo "---" && cat ~/hello-python-project/main.py', + timeoutMs: 10_000, + }, + }) + printResult('ls + cat', lsResult) + + console.info('\n🎉 Done! Python hello-world project created and executed via computer-use-mcp.') + } + finally { + await client.close().catch(() => {}) + } +} + +function printResult(label: string, result: unknown) { + if (!result || typeof result !== 'object') { + console.info(` [${label}] (no result)`) + return + } + + const r = result as Record + + // Print text content + if (Array.isArray(r.content)) { + for (const item of r.content) { + if (item && typeof item === 'object' && 'text' in item) { + console.info(` [${label}] ${(item as { text: string }).text}`) + } + } + } + + // Print structured content status + if (r.structuredContent && typeof r.structuredContent === 'object') { + const sc = r.structuredContent as Record + if (sc.status) { + console.info(` [${label}] status=${sc.status}`) + } + if (sc.output && typeof sc.output === 'object') { + const output = sc.output as Record + if (output.stdout) { + console.info(` [${label}] stdout: ${output.stdout}`) + } + if (output.stderr) { + console.info(` [${label}] stderr: ${output.stderr}`) + } + } + } +} + +main().catch((err) => { + console.error('❌ Fatal:', err instanceof Error ? err.message : String(err)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-airi-chat-observable.ts b/services/computer-use-mcp/src/bin/e2e-airi-chat-observable.ts new file mode 100644 index 000000000..74d44c508 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-airi-chat-observable.ts @@ -0,0 +1,1010 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' + +import type { AiriDebugSnapshotLike } from '../e2e/debug-targets' + +import { execFile, spawn } from 'node:child_process' +import { createWriteStream } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +import WebSocket from 'ws' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +import { hasCompletedChatTurn } from '../e2e/chat-turn' +import { + + isChatSurfaceTarget, + prioritizeInspectableAiriTargets, +} from '../e2e/debug-targets' +import { getProviderBootstrapConfig } from '../e2e/provider-bootstrap' + +interface DebugTarget { + id: string + title: string + type: string + url: string + webSocketDebuggerUrl?: string +} + +interface TimelineEntry { + at: string + event: string + detail?: Record +} + +interface ReportShape { + startedAt: string + finishedAt?: string + status: 'running' | 'completed' | 'failed' + prompt: string + reportDir: string + paths: { + reportPath: string + stageLogPath: string + mcpSessionRoot: string + auditLogPath?: string + screenshotsDir?: string + } + timeline: TimelineEntry[] + debugSnapshots: unknown[] + mcp: { + capabilities?: unknown + desktopState?: unknown + sessionTrace?: unknown + } + final?: { + providerConfigured?: boolean + providerId?: string + modelId?: string + messageCount?: number + lastMessageRole?: string + lastMessageText?: string + lastTurnOutput?: string + } + error?: string +} + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const repoDir = resolve(packageDir, '../..') +const preferredDebugPort = Number(env.AIRI_E2E_DEBUG_PORT || '9222') +const runId = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-') +const preferredProviderId = env.AIRI_E2E_PROVIDER?.trim() || 'github-models' +const preferredModelCandidates = Array.from(new Set( + (env.AIRI_E2E_MODELS?.trim() + ? env.AIRI_E2E_MODELS.split(',') + : [env.AIRI_E2E_MODEL?.trim() || 'openai/gpt-4o-mini', 'openai/gpt-4.1-mini']) + .map(model => model?.trim()) + .filter((model): model is string => Boolean(model)), +)) +const promptMarker = `airi-e2e-${runId.slice(-8)}` +// NOTICE: keep the default prompt ASCII-only. On macOS, injecting non-ASCII +// text through Quartz events can interact with the active IME composition state, +// which makes the follow-up Enter key commit composition instead of submitting +// the AIRI chat textarea. The prompt remains overrideable via AIRI_E2E_PROMPT. +const promptBaseText = env.AIRI_E2E_PROMPT?.trim() || 'Reply with one short sentence only: hello from AIRI desktop E2E.' +const promptText = `${promptBaseText} [${promptMarker}]` +const reportDir = resolve(packageDir, '.computer-use-mcp', 'reports', `airi-chat-observable-${runId}`) +const reportPath = resolve(reportDir, 'report.json') +const stageLogPath = resolve(reportDir, 'stage-tamagotchi.log') +const mcpSessionRoot = resolve(reportDir, 'computer-use-session') +const rootEnvPath = resolve(repoDir, '.env') + +const report: ReportShape = { + startedAt: new Date().toISOString(), + status: 'running', + prompt: promptText, + reportDir, + paths: { + reportPath, + stageLogPath, + mcpSessionRoot, + }, + timeline: [], + debugSnapshots: [], + mcp: {}, +} + +const execFileAsync = promisify(execFile) + +function addTimeline(event: string, detail?: Record) { + report.timeline.push({ + at: new Date().toISOString(), + event, + detail, + }) +} + +function parseCommandArgs(raw: string | undefined, fallback: string[]) { + if (!raw?.trim()) { + return fallback + } + + return raw + .split(/\s+/) + .map(item => item.trim()) + .filter(Boolean) +} + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') { + throw new Error(`${label} did not return an object result`) + } + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') { + throw new Error(`${label} missing structuredContent`) + } + + return structuredContent as Record +} + +function sleep(ms: number) { + return new Promise(resolvePromise => setTimeout(resolvePromise, ms)) +} + +async function withTimeout(label: string, task: Promise, timeoutMs: number) { + let timeoutHandle: NodeJS.Timeout | undefined + + try { + return await Promise.race([ + task, + new Promise((_resolvePromise, rejectPromise) => { + timeoutHandle = setTimeout(() => rejectPromise(new Error(`Timed out waiting for ${label}`)), timeoutMs) + }), + ]) + } + finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle) + } + } +} + +async function writeReport() { + report.finishedAt = new Date().toISOString() + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf-8') +} + +async function canListenOnPort(port: number) { + return await new Promise((resolvePromise) => { + const server = createServer() + server.once('error', () => { + resolvePromise(false) + }) + server.listen(port, '127.0.0.1', () => { + server.close(() => resolvePromise(true)) + }) + }) +} + +async function findAvailablePort(preferredPort: number, attempts = 20) { + for (let index = 0; index < attempts; index += 1) { + const candidate = preferredPort + index + if (await canListenOnPort(candidate)) { + return candidate + } + } + + throw new Error(`Could not find an available remote debug port starting from ${preferredPort}`) +} + +async function terminateExistingStageTamagotchiInstances() { + const patterns = [ + resolve(repoDir, 'apps', 'stage-tamagotchi'), + '@proj-airi/stage-tamagotchi', + resolve(repoDir, 'node_modules', '.pnpm', 'electron@'), + ] + + for (const pattern of patterns) { + await execFileAsync('pkill', ['-f', pattern]).catch(() => {}) + } + + await sleep(1_500) +} + +async function waitFor(label: string, task: () => Promise, timeoutMs = 60_000, intervalMs = 500) { + const startedAt = Date.now() + + while ((Date.now() - startedAt) < timeoutMs) { + const value = await task() + if (value !== undefined) { + return value + } + + await sleep(intervalMs) + } + + throw new Error(`Timed out waiting for ${label}`) +} + +function parseDotEnv(text: string) { + const values: Record = {} + + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#')) { + continue + } + + const separatorIndex = trimmed.indexOf('=') + if (separatorIndex <= 0) { + continue + } + + const key = trimmed.slice(0, separatorIndex).trim() + const rawValue = trimmed.slice(separatorIndex + 1).trim() + const unwrapped = rawValue.replace(/^['"]|['"]$/gu, '') + values[key] = unwrapped + } + + return values +} + +async function readRootEnvValues() { + try { + const raw = await readFile(rootEnvPath, 'utf-8') + return parseDotEnv(raw) + } + catch { + return {} + } +} + +class CdpClient { + private ws: any + private nextId = 0 + private pending = new Map void, reject: (error: Error) => void }>() + + static async connectToUrl(webSocketUrl: string, options: { enableRuntime?: boolean, enablePage?: boolean } = {}) { + const client = new CdpClient() + client.ws = new WebSocket(webSocketUrl) + + await new Promise((resolvePromise, rejectPromise) => { + const onOpen = () => resolvePromise() + const onError = (error: Error) => rejectPromise(error) + + client.ws.addEventListener('open', onOpen, { once: true }) + client.ws.addEventListener('error', onError, { once: true }) + }) + + client.ws.addEventListener('message', (event: { data: string }) => { + const payload = JSON.parse(event.data) + if (typeof payload.id === 'number') { + const pending = client.pending.get(payload.id) + if (!pending) { + return + } + + client.pending.delete(payload.id) + if (payload.error) { + pending.reject(new Error(String(payload.error.message || 'Unknown CDP error'))) + return + } + + pending.resolve(payload.result) + } + }) + + if (options.enableRuntime !== false) { + await client.send('Runtime.enable') + } + + if (options.enablePage !== false) { + await client.send('Page.enable') + } + + return client + } + + static async connect(target: DebugTarget) { + if (!target.webSocketDebuggerUrl) { + throw new Error(`Debug target ${target.title || target.id} does not expose webSocketDebuggerUrl`) + } + + return await CdpClient.connectToUrl(target.webSocketDebuggerUrl) + } + + async send(method: string, params?: Record) { + const id = ++this.nextId + const payload = { id, method, params } + + return await new Promise((resolvePromise, rejectPromise) => { + this.pending.set(id, { resolve: resolvePromise, reject: rejectPromise }) + this.ws.send(JSON.stringify(payload)) + }) + } + + async evaluate(expression: string): Promise { + const result = await this.send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + userGesture: true, + }) + + if (result?.exceptionDetails) { + const text = result.exceptionDetails.text || 'Runtime.evaluate exception' + throw new Error(String(text)) + } + + return result?.result?.value as T + } + + async close() { + if (this.ws?.readyState === 1) { + this.ws.close() + } + } +} + +async function listDebugTargets(browserWsUrl: string) { + const browserClient = await CdpClient.connectToUrl(browserWsUrl, { + enableRuntime: false, + enablePage: false, + }) + + try { + const result = await browserClient.send('Target.getTargets') as { targetInfos?: Array> } + const targetInfos = Array.isArray(result.targetInfos) ? result.targetInfos : [] + + return targetInfos + .filter(target => target.type === 'page') + .map((target) => { + const targetId = String(target.targetId || '') + return { + id: targetId, + title: String(target.title || ''), + type: String(target.type || ''), + url: String(target.url || ''), + webSocketDebuggerUrl: browserWsUrl.replace(/\/devtools\/browser\/[^/]+$/, `/devtools/page/${targetId}`), + } satisfies DebugTarget + }) + } + finally { + await browserClient.close().catch(() => {}) + } +} + +async function bringTargetToFront(client: CdpClient, label: string) { + await client.send('Page.bringToFront') + addTimeline('target-brought-to-front', { label }) + await sleep(750) +} + +async function getAiriDebugSnapshot(client: CdpClient) { + return await client.evaluate(`(() => { + const bridge = window.__AIRI_DEBUG__ + if (!bridge || typeof bridge.getSnapshot !== 'function') { + return undefined + } + + return bridge.getSnapshot() + })()`) +} + +async function waitForChatSurfaceReady(client: CdpClient, label: string) { + return await waitFor(label, async () => { + try { + const snapshot = await client.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + if (snapshot.dom?.hasTextarea) { + return snapshot + } + + return undefined + } + catch { + return undefined + } + }, 30_000, 250) +} + +async function findTargetWithAiriDebugBridge( + browserWsUrl: string, + label: string, + predicate?: (target: DebugTarget, snapshot: AiriDebugSnapshotLike) => boolean, +) { + return await waitFor(label, async () => { + const targets = prioritizeInspectableAiriTargets(await listDebugTargets(browserWsUrl).catch(() => [])) + + for (const target of targets) { + let client: CdpClient | undefined + + try { + client = await withTimeout( + `${label} connect ${target.title || target.url || target.id}`, + CdpClient.connect(target), + 2_500, + ) + const snapshot = await withTimeout( + `${label} snapshot ${target.title || target.url || target.id}`, + getAiriDebugSnapshot(client), + 2_500, + ) + if (!snapshot) { + continue + } + + if (predicate && !predicate(target, snapshot)) { + continue + } + + return { + target, + snapshot, + } + } + catch { + continue + } + finally { + await client?.close().catch(() => {}) + } + } + + return undefined + }, 90_000, 750) +} + +function summarizeMessageText(value: unknown) { + if (typeof value !== 'string') { + return '' + } + + const normalized = value.replace(/\s+/g, ' ').trim() + return normalized.length > 240 ? `${normalized.slice(0, 237)}...` : normalized +} + +let exitCode = 0 + +async function main() { + let stageProcess: ChildProcessWithoutNullStreams | undefined + let mcpClient: Client | undefined + let mainTargetClient: CdpClient | undefined + let chatTargetClient: CdpClient | undefined + let chatClientSharesMainTarget = false + let chatSurfaceMode: 'separate-window' | 'same-window-route' = 'separate-window' + let browserWsUrl: string | undefined + const debugPort = await findAvailablePort(preferredDebugPort) + const rootEnvValues = await readRootEnvValues() + const providerBootstrapConfig = getProviderBootstrapConfig({ + providerId: preferredProviderId, + processEnv: env, + dotenvValues: rootEnvValues, + }) + + try { + await mkdir(reportDir, { recursive: true }) + await mkdir(mcpSessionRoot, { recursive: true }) + + addTimeline('bootstrap', { reportDir, debugPort }) + await terminateExistingStageTamagotchiInstances() + addTimeline('terminated-stale-stage-tamagotchi-instances') + + const stageLogStream = createWriteStream(stageLogPath, { flags: 'a' }) + addTimeline('start-stage-tamagotchi') + + stageProcess = spawn('pnpm', ['-F', '@proj-airi/stage-tamagotchi', 'dev'], { + cwd: repoDir, + env: { + ...env, + APP_REMOTE_DEBUG: 'true', + APP_REMOTE_DEBUG_PORT: String(debugPort), + APP_REMOTE_DEBUG_NO_OPEN: 'true', + }, + stdio: 'pipe', + }) + + stageProcess.stdout.on('data', (chunk) => { + stageLogStream.write(chunk) + const match = chunk.toString('utf-8').match(/DevTools listening on (ws:\/\/\S+)/) + if (match?.[1]) { + browserWsUrl = match[1] + } + }) + stageProcess.stderr.on('data', (chunk) => { + stageLogStream.write(chunk) + const match = chunk.toString('utf-8').match(/DevTools listening on (ws:\/\/\S+)/) + if (match?.[1]) { + browserWsUrl = match[1] + } + }) + + stageProcess.on('exit', (code, signal) => { + addTimeline('stage-tamagotchi-exit', { + code: code ?? undefined, + signal: signal ?? undefined, + }) + }) + + const activeBrowserWsUrl = await waitFor('remote debug browser websocket', async () => { + return browserWsUrl + }, 120_000, 500) + addTimeline('remote-debug-browser-ready', { browserWsUrl: activeBrowserWsUrl, debugPort }) + + const mainTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'AIRI main target', + (_target, snapshot) => !String(snapshot.route || '').includes('/chat'), + ) + const mainTarget = mainTargetMatch.target + addTimeline('main-target-ready', { + title: mainTarget.title, + url: mainTarget.url, + route: mainTargetMatch.snapshot.route, + documentTitle: mainTargetMatch.snapshot.documentTitle, + }) + + mainTargetClient = await CdpClient.connect(mainTarget) + await bringTargetToFront(mainTargetClient, 'main') + + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start']) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'macos-local', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_OPENABLE_APPS: 'Terminal,Cursor,Google Chrome,Electron', + COMPUTER_USE_DENY_APPS: '1Password,Keychain,System Settings,Activity Monitor', + COMPUTER_USE_SESSION_TAG: `airi-e2e-${runId}`, + COMPUTER_USE_ALLOWED_BOUNDS: env.COMPUTER_USE_ALLOWED_BOUNDS || '0,0,2560,1600', + COMPUTER_USE_SESSION_ROOT: mcpSessionRoot, + }, + stderr: 'pipe', + }) + + mcpClient = new Client({ + name: '@proj-airi/computer-use-mcp-e2e-airi-chat', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk: { toString: (encoding: string) => string }) => { + const text = chunk.toString('utf-8').trim() + if (text) { + addTimeline('computer-use-mcp-stderr', { text }) + } + }) + + await mcpClient.connect(transport) + addTimeline('computer-use-mcp-connected') + + const capabilities = await mcpClient.callTool({ + name: 'desktop_get_capabilities', + arguments: {}, + }) + const capabilitiesData = requireStructuredContent(capabilities, 'desktop_get_capabilities') + report.mcp.capabilities = capabilitiesData + report.paths.auditLogPath = String((capabilitiesData.session as Record | undefined)?.auditLogPath || '') || undefined + report.paths.screenshotsDir = String((capabilitiesData.session as Record | undefined)?.screenshotsDir || '') || undefined + addTimeline('desktop-capabilities', { + executionMode: (capabilitiesData.executionTarget as Record | undefined)?.mode, + auditLogPath: report.paths.auditLogPath, + screenshotsDir: report.paths.screenshotsDir, + }) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'before-open-chat' }, + }) + addTimeline('screenshot-captured', { label: 'before-open-chat' }) + + try { + await withTimeout( + 'AIRI debug bridge openChat', + mainTargetClient.evaluate('window.__AIRI_DEBUG__.openChat()'), + 8_000, + ) + addTimeline('chat-open-requested', { mode: 'separate-window' }) + + const chatTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'Chat target', + (target, snapshot) => isChatSurfaceTarget(target, snapshot), + ) + const chatTarget = chatTargetMatch.target + addTimeline('chat-target-ready', { + title: chatTarget.title, + url: chatTarget.url, + route: chatTargetMatch.snapshot.route, + documentTitle: chatTargetMatch.snapshot.documentTitle, + mode: 'separate-window', + }) + + chatTargetClient = await CdpClient.connect(chatTarget) + await bringTargetToFront(chatTargetClient, 'chat') + const readyChatSnapshot = await waitForChatSurfaceReady(chatTargetClient, 'chat surface ready') + report.debugSnapshots.push(readyChatSnapshot) + addTimeline('chat-surface-ready', { + route: String(readyChatSnapshot.route || ''), + hasTextarea: Boolean(readyChatSnapshot.dom?.hasTextarea), + }) + } + catch (error) { + chatSurfaceMode = 'same-window-route' + addTimeline('chat-open-fallback', { + mode: 'same-window-route', + reason: error instanceof Error ? error.message : String(error), + }) + + await mainTargetClient.evaluate(`window.__AIRI_DEBUG__.navigateTo('/chat')`) + + await waitFor('chat route in main AIRI window', async () => { + try { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + const onChatRoute = String(snapshot.route || '').includes('/chat') + const hasTextarea = Boolean(snapshot.dom?.hasTextarea) + return onChatRoute && hasTextarea ? snapshot : undefined + } + catch { + return undefined + } + }, 30_000, 750) + + chatTargetClient = mainTargetClient + await bringTargetToFront(chatTargetClient, 'main-chat-fallback') + const readyChatSnapshot = await waitForChatSurfaceReady(chatTargetClient, 'fallback chat surface ready') + report.debugSnapshots.push(readyChatSnapshot) + addTimeline('chat-surface-ready', { + route: String(readyChatSnapshot.route || ''), + hasTextarea: Boolean(readyChatSnapshot.dom?.hasTextarea), + mode: 'same-window-route', + }) + chatClientSharesMainTarget = true + addTimeline('chat-target-ready', { + title: 'AIRI', + url: 'http://localhost:5173/#/chat', + mode: 'same-window-route', + }) + } + + const focusedDesktop = await mcpClient.callTool({ + name: 'desktop_focus_app', + arguments: { app: 'Electron' }, + }) + const focusedDesktopData = requireStructuredContent(focusedDesktop, 'desktop_focus_app') + addTimeline('desktop-focus-app', { + app: 'Electron', + status: focusedDesktopData.status, + }) + + const observation = await waitFor('Chat window observation', async () => { + const result = await mcpClient!.callTool({ + name: 'desktop_observe_windows', + arguments: { limit: 24 }, + }) + const data = requireStructuredContent(result, 'desktop_observe_windows') + const observationPayload = ((data.backendResult as Record | undefined)?.observation + || data.observation) as Record | undefined + const windows = Array.isArray(observationPayload?.windows) ? observationPayload.windows as Array> : [] + const frontmostAppName = String(observationPayload?.frontmostAppName || '') + const chatWindow = windows.find(window => String(window.title || '').includes('AIRI')) + if (!frontmostAppName.includes('Electron')) { + return undefined + } + if (!chatWindow) { + return undefined + } + + return { + full: data, + chatWindow, + } + }, 30_000, 1_000) + addTimeline('chat-window-observed', { + ...observation.chatWindow, + mode: chatSurfaceMode, + }) + + await chatTargetClient.evaluate('window.__AIRI_DEBUG__.clearEvents()') + const selectionSnapshot = await chatTargetClient.evaluate>(`window.__AIRI_DEBUG__.ensureConsciousnessSelection(${JSON.stringify({ + provider: preferredProviderId, + preferredModels: preferredModelCandidates, + providerConfig: providerBootstrapConfig, + })})`) + report.debugSnapshots.push(selectionSnapshot) + addTimeline('consciousness-selection-ready', { + providerId: String(selectionSnapshot.provider?.activeProvider || ''), + modelId: String(selectionSnapshot.provider?.activeModel || ''), + preferredProviderId, + preferredModelCandidates, + providerAvailable: Boolean(selectionSnapshot.provider?.providerAvailable), + providerBootstrapped: Boolean(providerBootstrapConfig), + }) + + if (preferredProviderId === 'github-models' && !selectionSnapshot.provider?.providerAvailable) { + throw new Error(`GitHub Models provider is unavailable before chat send. Checked .env at ${rootEnvPath} for bootstrap credentials, but AIRI still did not validate github-models.`) + } + + await chatTargetClient.evaluate('window.__AIRI_DEBUG__.clearEvents()') + const resetSnapshot = await chatTargetClient.evaluate>('window.__AIRI_DEBUG__.resetChatSession()') + report.debugSnapshots.push(resetSnapshot) + addTimeline('chat-session-reset', { + providerConfigured: Boolean(resetSnapshot.provider?.configured), + providerId: String(resetSnapshot.provider?.activeProvider || ''), + modelId: String(resetSnapshot.provider?.activeModel || ''), + messageCount: Number(resetSnapshot.chat?.messageCount || 0), + activeSessionId: String(resetSnapshot.chat?.activeSessionId || ''), + }) + + const focusState = await waitFor('chat textarea focus', async () => { + const state = await chatTargetClient!.evaluate>(`(() => { + window.focus() + const textarea = document.querySelector('textarea.ph-no-capture') + if (!(textarea instanceof HTMLTextAreaElement)) { + return { + ok: false, + reason: 'textarea-not-found', + } + } + + textarea.click() + textarea.focus() + + return { + ok: document.activeElement === textarea, + placeholder: textarea.getAttribute('placeholder'), + valueLength: textarea.value.length, + disabled: textarea.disabled, + readOnly: textarea.readOnly, + focusedTagName: document.activeElement?.tagName || '', + } + })()`) + + addTimeline('textarea-focus-poll', { + ok: Boolean(state.ok), + disabled: Boolean(state.disabled), + readOnly: Boolean(state.readOnly), + focusedTagName: String(state.focusedTagName || ''), + }) + + return state.ok ? state : undefined + }, 15_000, 250) + addTimeline('textarea-focused', { + placeholder: String(focusState.placeholder || ''), + valueLength: Number(focusState.valueLength || 0), + }) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'chat-before-type' }, + }) + addTimeline('screenshot-captured', { label: 'chat-before-type' }) + + const baselineMessageCount = Number(resetSnapshot.chat?.messageCount || 0) + + const typed = await mcpClient.callTool({ + name: 'desktop_type_text', + arguments: { + text: promptText, + pressEnter: false, + captureAfter: true, + }, + }) + const typedData = requireStructuredContent(typed, 'desktop_type_text') + addTimeline('desktop-type-text', { + status: typedData.status, + screenshotPath: (typedData.screenshot as Record | undefined)?.path, + }) + + const typedSnapshot = await waitFor('typed prompt to settle in textarea', async () => { + const typedState = await chatTargetClient!.evaluate>(`(() => { + const textarea = document.querySelector('textarea.ph-no-capture') + const value = textarea instanceof HTMLTextAreaElement ? textarea.value : '' + return { + value, + valueLength: value.length, + containsPromptMarker: value.includes(${JSON.stringify(promptMarker)}), + } + })()`) + + addTimeline('textarea-poll', { + valueLength: Number(typedState.valueLength || 0), + containsPromptMarker: Boolean(typedState.containsPromptMarker), + }) + + if (typedState.containsPromptMarker === true) { + return typedState + } + + return undefined + }, 10_000, 250) + addTimeline('textarea-filled', { + valueLength: Number(typedSnapshot.valueLength || 0), + }) + + const submit = await mcpClient.callTool({ + name: 'desktop_press_keys', + arguments: { + keys: ['enter'], + captureAfter: true, + }, + }) + const submitData = requireStructuredContent(submit, 'desktop_press_keys') + addTimeline('desktop-press-keys', { + status: submitData.status, + screenshotPath: (submitData.screenshot as Record | undefined)?.path, + }) + + const submittedSnapshot = await waitFor('chat submit', async () => { + const snapshot = await chatTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + report.debugSnapshots.push(snapshot) + + const messageCount = Number(snapshot.chat?.messageCount || 0) + const sending = Boolean(snapshot.chat?.sending) + const lastMessageRole = String(snapshot.chat?.lastMessage?.role || '') + const lastMessageText = String(snapshot.chat?.lastMessage?.text || '') + const recentEvents = Array.isArray(snapshot.chat?.recentEvents) ? snapshot.chat.recentEvents as Array> : [] + const sawBeforeSend = recentEvents.some(event => String(event?.type || '') === 'before-send') + + addTimeline('chat-submit-poll', { + sending, + messageCount, + lastMessageRole, + sawBeforeSend, + textareaValueLength: Number(snapshot.dom?.textareaValueLength || 0), + }) + + if (sending || sawBeforeSend) { + return snapshot + } + + if (messageCount > baselineMessageCount && lastMessageRole === 'user' && lastMessageText.includes(promptMarker)) { + return snapshot + } + + return undefined + }, 15_000, 500) + addTimeline('chat-submit-observed', { + sending: Boolean(submittedSnapshot.chat?.sending), + messageCount: Number(submittedSnapshot.chat?.messageCount || 0), + lastMessageRole: String(submittedSnapshot.chat?.lastMessage?.role || ''), + }) + + let capturedStreamingScreenshot = false + const finalSnapshot = await waitFor('chat completion or error', async () => { + const snapshot = await chatTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + report.debugSnapshots.push(snapshot) + + if (!capturedStreamingScreenshot && snapshot.chat?.sending && typeof snapshot.chat?.streamingText === 'string' && snapshot.chat.streamingText.trim().length > 0) { + capturedStreamingScreenshot = true + await mcpClient!.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'chat-during-stream' }, + }) + addTimeline('screenshot-captured', { + label: 'chat-during-stream', + streamingLength: snapshot.chat.streamingText.length, + }) + } + + const messageCount = Number(snapshot.chat?.messageCount || 0) + const sending = Boolean(snapshot.chat?.sending) + const lastMessageRole = String(snapshot.chat?.lastMessage?.role || '') + const hasTurnCompletion = hasCompletedChatTurn(snapshot) + const recentEvents = Array.isArray(snapshot.chat?.recentEvents) + ? snapshot.chat.recentEvents as Array> + : [] + const abortedByUser = recentEvents.some(event => String(event?.type || '') === 'chat-abort-requested') + + addTimeline('chat-completion-poll', { + sending, + messageCount, + streamingLength: Number(snapshot.chat?.streamingText?.length || 0), + lastMessageRole, + turnCompleted: hasTurnCompletion, + toolCallCount: Number(snapshot.chat?.lastTurnComplete?.toolCallCount || 0), + toolResultCount: Number(snapshot.chat?.lastTurnComplete?.toolResultCount || 0), + abortedByUser, + }) + + if (!sending && messageCount > baselineMessageCount && hasTurnCompletion) { + return snapshot + } + + if (!sending && lastMessageRole === 'error') { + return snapshot + } + + if (!sending && abortedByUser) { + return snapshot + } + + return undefined + }, 90_000, 1_000) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'chat-final' }, + }) + addTimeline('screenshot-captured', { label: 'chat-final' }) + + const desktopState = await mcpClient.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + report.mcp.desktopState = requireStructuredContent(desktopState, 'desktop_get_state') + + const sessionTrace = await mcpClient.callTool({ + name: 'desktop_get_session_trace', + arguments: { limit: 200 }, + }) + report.mcp.sessionTrace = requireStructuredContent(sessionTrace, 'desktop_get_session_trace') + + report.final = { + providerConfigured: Boolean(finalSnapshot.provider?.configured), + providerId: String(finalSnapshot.provider?.activeProvider || ''), + modelId: String(finalSnapshot.provider?.activeModel || ''), + messageCount: Number(finalSnapshot.chat?.messageCount || 0), + lastMessageRole: String(finalSnapshot.chat?.lastMessage?.role || ''), + lastMessageText: summarizeMessageText(finalSnapshot.chat?.lastMessage?.text), + lastTurnOutput: summarizeMessageText(finalSnapshot.chat?.lastTurnComplete?.outputText), + } + + if (report.final.lastMessageRole === 'error') { + throw new Error(`AIRI chat failed on ${report.final.providerId}/${report.final.modelId}: ${report.final.lastMessageText || 'unknown error'}`) + } + + if (report.paths.auditLogPath) { + const audit = await readFile(report.paths.auditLogPath, 'utf-8').catch(() => '') + addTimeline('audit-log-summary', { + lineCount: audit ? audit.trim().split('\n').filter(Boolean).length : 0, + }) + } + + report.status = 'completed' + await writeReport() + + console.info(JSON.stringify({ + ok: true, + reportPath, + providerConfigured: report.final.providerConfigured, + providerId: report.final.providerId, + modelId: report.final.modelId, + lastMessageRole: report.final.lastMessageRole, + lastMessageText: report.final.lastMessageText, + lastTurnOutput: report.final.lastTurnOutput, + auditLogPath: report.paths.auditLogPath, + screenshotsDir: report.paths.screenshotsDir, + }, null, 2)) + } + catch (error) { + report.status = 'failed' + report.error = error instanceof Error ? error.stack || error.message : String(error) + addTimeline('failure', { error: report.error }) + await writeReport() + console.error(report.error) + exitCode = 1 + } + finally { + if (chatTargetClient && !chatClientSharesMainTarget) { + await chatTargetClient.close().catch(() => {}) + } + await mainTargetClient?.close().catch(() => {}) + await mcpClient?.close().catch(() => {}) + + if (stageProcess && !stageProcess.killed) { + stageProcess.kill('SIGINT') + await sleep(1_500) + if (stageProcess.exitCode == null) { + stageProcess.kill('SIGTERM') + } + } + + await writeReport().catch(() => {}) + } +} + +main().finally(() => { + exit(exitCode) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-airi-chat-terminal-self-acquire.ts b/services/computer-use-mcp/src/bin/e2e-airi-chat-terminal-self-acquire.ts new file mode 100644 index 000000000..b949e6787 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-airi-chat-terminal-self-acquire.ts @@ -0,0 +1,1232 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' + +import type { AiriDebugSnapshotLike, DebugTargetLike } from '../e2e/debug-targets' + +import { Buffer } from 'node:buffer' +import { execFile, spawn } from 'node:child_process' +import { createWriteStream } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +import WebSocket from 'ws' + +import { hasCompletedChatTurn } from '../e2e/chat-turn' +import { + isChatSurfaceTarget, + prioritizeInspectableAiriTargets, +} from '../e2e/debug-targets' +import { getProviderBootstrapConfig, resolvePreferredChatProviderId } from '../e2e/provider-bootstrap' + +interface DebugTarget extends DebugTargetLike { + webSocketDebuggerUrl?: string +} + +interface TimelineEntry { + at: string + event: string + detail?: Record +} + +interface ReportShape { + startedAt: string + finishedAt?: string + status: 'running' | 'completed' | 'failed' + prompt: string + reportDir: string + paths: { + reportPath: string + demoSummaryPath: string + screenshotsDir: string + stageLogPath: string + userDataDir: string + mcpConfigPath: string + mcpSessionRoot: string + } + timeline: TimelineEntry[] + debugSnapshots: unknown[] + internalMcp: { + tools?: unknown + ptyStatus?: unknown + desktopState?: unknown + sessionTrace?: unknown + } + final?: { + providerConfigured?: boolean + providerId?: string + modelId?: string + messageCount?: number + lastMessageRole?: string + lastMessageText?: string + lastTurnOutput?: string + ptySessionId?: string + demoSummaryText?: string + screenshotPaths?: string[] + } + error?: string +} + +const execFileAsync = promisify(execFile) +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const repoDir = resolve(packageDir, '../..') +const preferredDebugPort = Number(env.AIRI_E2E_DEBUG_PORT || '9222') +const runId = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-') +const repoChangesCommand = 'git diff --stat -- services/computer-use-mcp packages/stage-ui apps/stage-tamagotchi' +const summaryMarker = 'Terminal self-acquire demo complete.' +const promptText = [ + `Use MCP tools to validate the AIRI repository at ${repoDir}.`, + 'Call the real workflow, do not narrate or simulate tool results.', + '1. Call computer_use::workflow_validate_workspace with these exact arguments:', + ` - projectPath: ${repoDir}`, + ' - ideApp: Visual Studio Code', + ` - changesCommand: ${repoChangesCommand}`, + ' - checkCommand: vim --version', + ' - autoApprove: true', + '2. Do not call pty_create manually. The workflow should acquire PTY by itself if needed.', + '3. Do not call any more tools after the workflow returns.', + '4. Reply in plain text with 3 short bullet points for a management audience.', + '5. Mention that validation started on exec, the workflow self-acquired PTY for the interactive validation command, and the workflow completed successfully.', +].join('\n') +const reportDir = resolve(packageDir, '.computer-use-mcp', 'reports', `airi-chat-terminal-self-acquire-${runId}`) +const reportPath = resolve(reportDir, 'report.json') +const demoSummaryPath = resolve(reportDir, 'demo-summary.md') +const screenshotsDir = resolve(reportDir, 'screenshots') +const stageLogPath = resolve(reportDir, 'stage-tamagotchi.log') +const userDataDir = resolve(reportDir, 'stage-user-data') +const mcpConfigPath = resolve(userDataDir, 'mcp.json') +const mcpSessionRoot = resolve(reportDir, 'computer-use-session') +const rootEnvPath = resolve(repoDir, '.env') + +const report: ReportShape = { + startedAt: new Date().toISOString(), + status: 'running', + prompt: promptText, + reportDir, + paths: { + reportPath, + demoSummaryPath, + screenshotsDir, + stageLogPath, + userDataDir, + mcpConfigPath, + mcpSessionRoot, + }, + timeline: [], + debugSnapshots: [], + internalMcp: {}, +} + +function assert(condition: boolean, message: string): asserts condition { + if (!condition) { + throw new Error(`Assertion failed: ${message}`) + } +} + +function addTimeline(event: string, detail?: Record) { + report.timeline.push({ + at: new Date().toISOString(), + event, + detail, + }) +} + +function parseDotEnv(text: string) { + const values: Record = {} + + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#')) { + continue + } + + const separatorIndex = trimmed.indexOf('=') + if (separatorIndex <= 0) { + continue + } + + const key = trimmed.slice(0, separatorIndex).trim() + const rawValue = trimmed.slice(separatorIndex + 1).trim() + const unwrapped = rawValue.replace(/^['"]|['"]$/gu, '') + values[key] = unwrapped + } + + return values +} + +async function readRootEnvValues() { + try { + const raw = await readFile(rootEnvPath, 'utf-8') + return parseDotEnv(raw) + } + catch { + return {} + } +} + +function sleep(ms: number) { + return new Promise(resolvePromise => setTimeout(resolvePromise, ms)) +} + +async function withTimeout(label: string, task: Promise, timeoutMs: number) { + let timeoutHandle: NodeJS.Timeout | undefined + + try { + return await Promise.race([ + task, + new Promise((_resolvePromise, rejectPromise) => { + timeoutHandle = setTimeout(() => rejectPromise(new Error(`Timed out waiting for ${label}`)), timeoutMs) + }), + ]) + } + finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle) + } + } +} + +async function writeReport() { + report.finishedAt = new Date().toISOString() + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf-8') +} + +async function canListenOnPort(port: number) { + return await new Promise((resolvePromise) => { + const server = createServer() + server.once('error', () => { + resolvePromise(false) + }) + server.listen(port, '127.0.0.1', () => { + server.close(() => resolvePromise(true)) + }) + }) +} + +async function findAvailablePort(preferredPort: number, attempts = 20) { + for (let index = 0; index < attempts; index += 1) { + const candidate = preferredPort + index + if (await canListenOnPort(candidate)) { + return candidate + } + } + + throw new Error(`Could not find an available remote debug port starting from ${preferredPort}`) +} + +async function terminateExistingStageTamagotchiInstances() { + const patterns = [ + resolve(repoDir, 'apps', 'stage-tamagotchi'), + '@proj-airi/stage-tamagotchi', + resolve(repoDir, 'node_modules', '.pnpm', 'electron@'), + ] + + for (const pattern of patterns) { + await execFileAsync('pkill', ['-f', pattern]).catch(() => {}) + } + + await sleep(1_500) +} + +async function waitFor(label: string, task: () => Promise, timeoutMs = 60_000, intervalMs = 500) { + const startedAt = Date.now() + + while ((Date.now() - startedAt) < timeoutMs) { + const value = await task() + if (value !== undefined) { + return value + } + + await sleep(intervalMs) + } + + throw new Error(`Timed out waiting for ${label}`) +} + +function summarizeMessageText(value: unknown) { + if (typeof value !== 'string') { + return '' + } + + const normalized = value.replace(/\s+/g, ' ').trim() + return normalized.length > 280 ? `${normalized.slice(0, 277)}...` : normalized +} + +function getPreferredModels(providerId: string) { + const requested = env.AIRI_E2E_MODELS?.trim() + ? env.AIRI_E2E_MODELS.split(',') + : [env.AIRI_E2E_MODEL?.trim() || ''] + + const explicit = requested + .map(model => model?.trim()) + .filter((model): model is string => Boolean(model)) + if (explicit.length > 0) { + return explicit + } + + if (providerId === 'google-generative-ai') { + return ['gemini-2.5-flash', 'models/gemini-2.5-flash', 'gemini-2.5-pro'] + } + + return ['openai/gpt-4o-mini', 'openai/gpt-4.1-mini'] +} + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') { + throw new Error(`${label} did not return an object result`) + } + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') { + throw new Error(`${label} missing structuredContent`) + } + + return structuredContent as Record +} + +class CdpClient { + private ws: any + private nextId = 0 + private pending = new Map void, reject: (error: Error) => void }>() + + static async connectToUrl(webSocketUrl: string, options: { enableRuntime?: boolean, enablePage?: boolean } = {}) { + const client = new CdpClient() + client.ws = new WebSocket(webSocketUrl) + + await new Promise((resolvePromise, rejectPromise) => { + const onOpen = () => resolvePromise() + const onError = (error: Error) => rejectPromise(error) + + client.ws.addEventListener('open', onOpen, { once: true }) + client.ws.addEventListener('error', onError, { once: true }) + }) + + client.ws.addEventListener('message', (event: { data: string }) => { + const payload = JSON.parse(event.data) + if (typeof payload.id === 'number') { + const pending = client.pending.get(payload.id) + if (!pending) { + return + } + + client.pending.delete(payload.id) + if (payload.error) { + pending.reject(new Error(String(payload.error.message || 'Unknown CDP error'))) + return + } + + pending.resolve(payload.result) + } + }) + + if (options.enableRuntime !== false) { + await client.send('Runtime.enable') + } + + if (options.enablePage !== false) { + await client.send('Page.enable') + } + + return client + } + + static async connect(target: DebugTarget) { + if (!target.webSocketDebuggerUrl) { + throw new Error(`Debug target ${target.title || target.id} does not expose webSocketDebuggerUrl`) + } + + return await CdpClient.connectToUrl(target.webSocketDebuggerUrl) + } + + async send(method: string, params?: Record) { + const id = ++this.nextId + const payload = { id, method, params } + + return await new Promise((resolvePromise, rejectPromise) => { + this.pending.set(id, { resolve: resolvePromise, reject: rejectPromise }) + this.ws.send(JSON.stringify(payload)) + }) + } + + async evaluate(expression: string): Promise { + const result = await this.send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + userGesture: true, + }) + + if (result?.exceptionDetails) { + const text = result.exceptionDetails.text || 'Runtime.evaluate exception' + throw new Error(String(text)) + } + + return result?.result?.value as T + } + + async close() { + if (this.ws?.readyState === 1) { + this.ws.close() + } + } +} + +function toSafeFileStem(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + || 'capture' +} + +async function captureChatScreenshot(client: CdpClient, label: string) { + await mkdir(screenshotsDir, { recursive: true }) + const filePath = resolve(screenshotsDir, `${String(report.timeline.length).padStart(3, '0')}-${toSafeFileStem(label)}.png`) + const result = await client.send('Page.captureScreenshot', { + format: 'png', + captureBeyondViewport: true, + fromSurface: true, + }) as { data?: string } + + assert(typeof result.data === 'string' && result.data.length > 0, `Page.captureScreenshot returned no data for ${label}`) + await writeFile(filePath, Buffer.from(result.data, 'base64')) + addTimeline('chat-screenshot-captured', { + label, + path: filePath, + }) + return filePath +} + +async function writeDemoSummary(params: { + providerId: string + modelId: string + ptySessionId: string + recentSurfaceDecision: Record + auditDeltaCount: number + newTraceCount: number + screenContent: string + demoSummaryText?: string + screenshotPaths: string[] +}) { + const lines = [ + '# AIRI Terminal Self-Acquire Demo', + '', + `Report: ${reportPath}`, + `Provider: ${params.providerId}`, + `Model: ${params.modelId}`, + `PTY session: ${params.ptySessionId}`, + '', + '## What This Demonstrates', + '- AIRI started on the normal workflow terminal path (`exec`).', + '- The workflow recognized that the validation step needed an interactive terminal and self-acquired PTY inside the workflow.', + `- The validation step executed on PTY session \`${params.ptySessionId}\` without an outward reroute.`, + `- Verification evidence included ${params.auditDeltaCount} PTY audit entries and ${params.newTraceCount} new trace entries.`, + '', + '## Surface Decision', + `- Surface: ${String(params.recentSurfaceDecision.surface || '')}`, + `- Transport: ${String(params.recentSurfaceDecision.transport || '')}`, + `- Reason: ${String(params.recentSurfaceDecision.reason || '')}`, + '', + '## PTY Evidence', + '```text', + params.screenContent, + '```', + ] + + if (params.demoSummaryText?.trim()) { + lines.push('', '## AIRI Final Visible Summary', '', params.demoSummaryText.trim()) + } + + if (params.screenshotPaths.length > 0) { + lines.push('', '## Screenshots') + for (const screenshotPath of params.screenshotPaths) { + lines.push(`- ${screenshotPath}`) + } + } + + await writeFile(demoSummaryPath, `${lines.join('\n')}\n`, 'utf-8') +} + +async function listDebugTargets(browserWsUrl: string) { + const browserClient = await CdpClient.connectToUrl(browserWsUrl, { + enableRuntime: false, + enablePage: false, + }) + + try { + const result = await browserClient.send('Target.getTargets') as { targetInfos?: Array> } + const targetInfos = Array.isArray(result.targetInfos) ? result.targetInfos : [] + + return targetInfos + .filter(target => target.type === 'page') + .map((target) => { + const targetId = String(target.targetId || '') + return { + id: targetId, + title: String(target.title || ''), + type: String(target.type || ''), + url: String(target.url || ''), + webSocketDebuggerUrl: browserWsUrl.replace(/\/devtools\/browser\/[^/]+$/, `/devtools/page/${targetId}`), + } satisfies DebugTarget + }) + } + finally { + await browserClient.close().catch(() => {}) + } +} + +async function bringTargetToFront(client: CdpClient, label: string) { + await client.send('Page.bringToFront') + addTimeline('target-brought-to-front', { label }) + await sleep(750) +} + +async function getAiriDebugSnapshot(client: CdpClient) { + return await client.evaluate(`(() => { + const bridge = window.__AIRI_DEBUG__ + if (!bridge || typeof bridge.getSnapshot !== 'function') { + return undefined + } + + return bridge.getSnapshot() + })()`) +} + +async function callAiriDebugBridge(client: CdpClient, method: string, args: unknown[] = []) { + return await client.evaluate(`(async () => { + const bridge = window.__AIRI_DEBUG__ + if (!bridge) { + throw new Error('AIRI debug bridge is unavailable') + } + + const fn = bridge[${JSON.stringify(method)}] + if (typeof fn !== 'function') { + throw new Error('AIRI debug bridge method is unavailable: ${method}') + } + + return await fn.apply(bridge, ${JSON.stringify(args)}) + })()`) +} + +async function waitForChatSurfaceReady(client: CdpClient, label: string) { + return await waitFor(label, async () => { + try { + const snapshot = await callAiriDebugBridge>(client, 'getSnapshot') + if (snapshot.dom?.hasTextarea) { + return snapshot + } + + return undefined + } + catch { + return undefined + } + }, 30_000, 250) +} + +async function findTargetWithAiriDebugBridge( + browserWsUrl: string, + label: string, + predicate?: (target: DebugTarget, snapshot: AiriDebugSnapshotLike) => boolean, +) { + return await waitFor(label, async () => { + const targets = prioritizeInspectableAiriTargets(await listDebugTargets(browserWsUrl).catch(() => [])) + + for (const target of targets) { + let client: CdpClient | undefined + + try { + client = await withTimeout( + `${label} connect ${target.title || target.url || target.id}`, + CdpClient.connect(target), + 2_500, + ) + const snapshot = await withTimeout( + `${label} snapshot ${target.title || target.url || target.id}`, + getAiriDebugSnapshot(client), + 2_500, + ) + if (!snapshot) { + continue + } + + if (predicate && !predicate(target, snapshot)) { + continue + } + + return { + target, + snapshot, + } + } + catch { + continue + } + finally { + await client?.close().catch(() => {}) + } + } + + return undefined + }, 90_000, 750) +} + +async function prepareMcpConfig() { + await mkdir(userDataDir, { recursive: true }) + await mkdir(mcpSessionRoot, { recursive: true }) + + const config = { + mcpServers: { + computer_use: { + command: 'pnpm', + args: ['-F', '@proj-airi/computer-use-mcp', 'start'], + cwd: repoDir, + enabled: true, + env: { + COMPUTER_USE_EXECUTOR: 'dry-run', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_SESSION_TAG: `airi-chat-terminal-self-acquire-${runId}`, + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,1920,1080', + COMPUTER_USE_OPENABLE_APPS: 'Finder,Terminal,Visual Studio Code,Cursor', + COMPUTER_USE_SESSION_ROOT: mcpSessionRoot, + }, + }, + }, + } + + await writeFile(mcpConfigPath, `${JSON.stringify(config, null, 2)}\n`, 'utf-8') + addTimeline('prepared-mcp-config', { + mcpConfigPath, + mcpSessionRoot, + }) +} + +function extractTextContent(result: Record) { + const content = Array.isArray(result.content) ? result.content : [] + return content + .filter(item => item && typeof item === 'object' && item.type === 'text' && typeof item.text === 'string') + .map(item => String(item.text)) + .join('\n') +} + +function traceHasTerminalExecCommand(trace: Array>, commandFragment: string) { + return trace.some((entry) => { + const action = entry.action as Record | undefined + const input = action?.input as Record | undefined + return action?.kind === 'terminal_exec' + && typeof input?.command === 'string' + && input.command.includes(commandFragment) + }) +} + +let exitCode = 0 + +async function main() { + let stageProcess: ChildProcessWithoutNullStreams | undefined + let mainTargetClient: CdpClient | undefined + let chatTargetClient: CdpClient | undefined + let chatClientSharesMainTarget = false + let browserWsUrl: string | undefined + const screenshotPaths: string[] = [] + const debugPort = await findAvailablePort(preferredDebugPort) + const rootEnvValues = await readRootEnvValues() + const resolvedPreferredProviderId = resolvePreferredChatProviderId({ + requestedProviderId: env.AIRI_E2E_PROVIDER?.trim(), + processEnv: env, + dotenvValues: rootEnvValues, + }) + const providerAttemptOrder = Array.from(new Set([ + env.AIRI_E2E_PROVIDER?.trim(), + resolvedPreferredProviderId, + 'google-generative-ai', + 'github-models', + ].filter((providerId): providerId is string => Boolean(providerId)))) + + try { + await mkdir(reportDir, { recursive: true }) + await prepareMcpConfig() + + addTimeline('bootstrap', { reportDir, debugPort, userDataDir }) + await terminateExistingStageTamagotchiInstances() + addTimeline('terminated-stale-stage-tamagotchi-instances') + + const stageLogStream = createWriteStream(stageLogPath, { flags: 'a' }) + addTimeline('start-stage-tamagotchi') + + stageProcess = spawn('pnpm', ['-F', '@proj-airi/stage-tamagotchi', 'dev'], { + cwd: repoDir, + env: { + ...env, + APP_REMOTE_DEBUG: 'true', + APP_REMOTE_DEBUG_PORT: String(debugPort), + APP_REMOTE_DEBUG_NO_OPEN: 'true', + APP_USER_DATA_PATH: userDataDir, + }, + stdio: 'pipe', + }) + + stageProcess.stdout.on('data', (chunk) => { + stageLogStream.write(chunk) + const match = chunk.toString('utf-8').match(/DevTools listening on (ws:\/\/\S+)/) + if (match?.[1]) { + browserWsUrl = match[1] + } + }) + stageProcess.stderr.on('data', (chunk) => { + stageLogStream.write(chunk) + const match = chunk.toString('utf-8').match(/DevTools listening on (ws:\/\/\S+)/) + if (match?.[1]) { + browserWsUrl = match[1] + } + }) + + stageProcess.on('exit', (code, signal) => { + addTimeline('stage-tamagotchi-exit', { + code: code ?? undefined, + signal: signal ?? undefined, + }) + }) + + const activeBrowserWsUrl = await waitFor('remote debug browser websocket', async () => { + return browserWsUrl + }, 120_000, 500) + addTimeline('remote-debug-browser-ready', { browserWsUrl: activeBrowserWsUrl, debugPort }) + + const mainTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'AIRI main target', + (_target, snapshot) => !String(snapshot.route || '').includes('/chat'), + ) + const mainTarget = mainTargetMatch.target + addTimeline('main-target-ready', { + title: mainTarget.title, + url: mainTarget.url, + route: mainTargetMatch.snapshot.route, + documentTitle: mainTargetMatch.snapshot.documentTitle, + }) + + mainTargetClient = await CdpClient.connect(mainTarget) + await bringTargetToFront(mainTargetClient, 'main') + + try { + await withTimeout( + 'AIRI debug bridge openChat', + callAiriDebugBridge(mainTargetClient, 'openChat'), + 8_000, + ) + addTimeline('chat-open-requested', { mode: 'separate-window' }) + + const chatTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'Chat target', + (target, snapshot) => isChatSurfaceTarget(target, snapshot), + ) + const chatTarget = chatTargetMatch.target + addTimeline('chat-target-ready', { + title: chatTarget.title, + url: chatTarget.url, + route: chatTargetMatch.snapshot.route, + documentTitle: chatTargetMatch.snapshot.documentTitle, + mode: 'separate-window', + }) + + chatTargetClient = await CdpClient.connect(chatTarget) + await bringTargetToFront(chatTargetClient, 'chat') + const readyChatSnapshot = await waitForChatSurfaceReady(chatTargetClient, 'chat surface ready') + report.debugSnapshots.push(readyChatSnapshot) + addTimeline('chat-surface-ready', { + route: String(readyChatSnapshot.route || ''), + hasTextarea: Boolean(readyChatSnapshot.dom?.hasTextarea), + }) + } + catch (error) { + addTimeline('chat-open-fallback', { + mode: 'same-window-route', + reason: error instanceof Error ? error.message : String(error), + }) + + await mainTargetClient.close().catch(() => {}) + const refreshedMainTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'AIRI main target (fallback refresh)', + (_target, snapshot) => !String(snapshot.route || '').includes('/chat'), + ) + mainTargetClient = await CdpClient.connect(refreshedMainTargetMatch.target) + await bringTargetToFront(mainTargetClient, 'main-fallback-refresh') + + await callAiriDebugBridge(mainTargetClient, 'navigateTo', ['/chat']) + + await waitFor('chat route in main AIRI window', async () => { + try { + const snapshot = await callAiriDebugBridge>(mainTargetClient!, 'getSnapshot') + const onChatRoute = String(snapshot.route || '').includes('/chat') + const hasTextarea = Boolean(snapshot.dom?.hasTextarea) + return onChatRoute && hasTextarea ? snapshot : undefined + } + catch { + return undefined + } + }, 30_000, 750) + + chatTargetClient = mainTargetClient + await bringTargetToFront(chatTargetClient, 'main-chat-fallback') + const readyChatSnapshot = await waitForChatSurfaceReady(chatTargetClient, 'fallback chat surface ready') + report.debugSnapshots.push(readyChatSnapshot) + addTimeline('chat-surface-ready', { + route: String(readyChatSnapshot.route || ''), + hasTextarea: Boolean(readyChatSnapshot.dom?.hasTextarea), + mode: 'same-window-route', + }) + chatClientSharesMainTarget = true + addTimeline('chat-target-ready', { + title: 'AIRI', + url: 'http://localhost:5173/#/chat', + mode: 'same-window-route', + }) + } + + screenshotPaths.push(await captureChatScreenshot(chatTargetClient, 'chat-ready')) + await callAiriDebugBridge(chatTargetClient, 'clearEvents') + let selectionSnapshot: Record | undefined + let selectedProviderId = '' + let selectedModelCandidates: string[] = [] + + for (const providerId of providerAttemptOrder) { + const providerBootstrapConfig = getProviderBootstrapConfig({ + providerId, + processEnv: env, + dotenvValues: rootEnvValues, + }) + const candidateModels = getPreferredModels(providerId) + const candidateSnapshot = await callAiriDebugBridge>(chatTargetClient, 'ensureConsciousnessSelection', [{ + provider: providerId, + preferredModels: candidateModels, + providerConfig: providerBootstrapConfig, + }]) + + report.debugSnapshots.push(candidateSnapshot) + addTimeline('consciousness-selection-attempt', { + providerId, + modelCandidates: candidateModels, + resolvedModelId: String(candidateSnapshot.provider?.activeModel || ''), + providerAvailable: Boolean(candidateSnapshot.provider?.providerAvailable), + providerBootstrapped: Boolean(providerBootstrapConfig), + }) + + if (candidateSnapshot.provider?.providerAvailable) { + selectionSnapshot = candidateSnapshot + selectedProviderId = providerId + selectedModelCandidates = candidateModels + break + } + } + + if (!selectionSnapshot) { + throw new Error(`No chat provider is available for AIRI chat E2E. Tried: ${providerAttemptOrder.join(', ')}`) + } + + addTimeline('consciousness-selection-ready', { + providerId: selectedProviderId, + modelId: String(selectionSnapshot.provider?.activeModel || ''), + modelCandidates: selectedModelCandidates, + providerAvailable: Boolean(selectionSnapshot.provider?.providerAvailable), + }) + + await callAiriDebugBridge(chatTargetClient, 'clearEvents') + const resetSnapshot = await callAiriDebugBridge>(chatTargetClient, 'resetChatSession') + report.debugSnapshots.push(resetSnapshot) + addTimeline('chat-session-reset', { + providerConfigured: Boolean(resetSnapshot.provider?.configured), + providerId: String(resetSnapshot.provider?.activeProvider || ''), + modelId: String(resetSnapshot.provider?.activeModel || ''), + messageCount: Number(resetSnapshot.chat?.messageCount || 0), + activeSessionId: String(resetSnapshot.chat?.activeSessionId || ''), + }) + + const availableTools = await waitFor('computer_use tools inside AIRI', async () => { + try { + const tools = await callAiriDebugBridge>>(chatTargetClient!, 'listMcpTools') + const names = new Set(tools.map(tool => String(tool.name || ''))) + const requiredTools = [ + 'computer_use::workflow_validate_workspace', + 'computer_use::pty_get_status', + 'computer_use::pty_read_screen', + 'computer_use::pty_destroy', + 'computer_use::desktop_get_state', + 'computer_use::desktop_get_session_trace', + ] + const ready = requiredTools.every(name => names.has(name)) + addTimeline('mcp-tool-list-poll', { + toolCount: tools.length, + ready, + }) + return ready ? tools : undefined + } + catch { + return undefined + } + }, 120_000, 1_000) + report.internalMcp.tools = availableTools + addTimeline('internal-mcp-ready', { + toolCount: Array.isArray(availableTools) ? availableTools.length : 0, + }) + + const ptyStatusResult = await callAiriDebugBridge>(chatTargetClient, 'callMcpTool', [{ + name: 'computer_use::pty_get_status', + arguments: {}, + }]) + const ptyStatusData = requireStructuredContent(ptyStatusResult, 'computer_use::pty_get_status') + report.internalMcp.ptyStatus = ptyStatusData + addTimeline('pty-status-probed', { + ptyAvailable: Boolean(ptyStatusData.ptyAvailable), + error: typeof ptyStatusData.error === 'string' ? ptyStatusData.error : undefined, + sessionCount: Array.isArray(ptyStatusData.sessions) ? ptyStatusData.sessions.length : 0, + }) + assert( + ptyStatusData.ptyAvailable === true, + `pty_get_status expected ptyAvailable=true, got ${String(ptyStatusData.ptyAvailable)}${typeof ptyStatusData.error === 'string' ? ` (${ptyStatusData.error})` : ''}`, + ) + + const baselineStateResult = await callAiriDebugBridge>(chatTargetClient, 'callMcpTool', [{ + name: 'computer_use::desktop_get_state', + arguments: {}, + }]) + const baselineState = requireStructuredContent(baselineStateResult, 'computer_use::desktop_get_state (baseline)') + const baselineRunState = (baselineState.runState || {}) as Record + const baselineAuditCount = Array.isArray(baselineRunState.ptyAuditLog) ? baselineRunState.ptyAuditLog.length : 0 + const baselinePtySessionCount = Array.isArray(baselineRunState.ptySessions) ? baselineRunState.ptySessions.length : 0 + + const baselineTraceResult = await callAiriDebugBridge>(chatTargetClient, 'callMcpTool', [{ + name: 'computer_use::desktop_get_session_trace', + arguments: { limit: 200 }, + }]) + const baselineTrace = requireStructuredContent(baselineTraceResult, 'computer_use::desktop_get_session_trace (baseline)') + const baselineTraceCount = Array.isArray(baselineTrace.trace) ? baselineTrace.trace.length : 0 + addTimeline('baseline-state-captured', { + baselineAuditCount, + baselineTraceCount, + baselinePtySessionCount, + }) + + const baselineMessageCount = Number(resetSnapshot.chat?.messageCount || 0) + await callAiriDebugBridge(chatTargetClient, 'sendChatPrompt', [promptText]) + addTimeline('chat-send-dispatched', { + baselineMessageCount, + }) + + await waitFor('chat submission', async () => { + const snapshot = await callAiriDebugBridge>(chatTargetClient!, 'getSnapshot') + report.debugSnapshots.push(snapshot) + + const messageCount = Number(snapshot.chat?.messageCount || 0) + const sending = Boolean(snapshot.chat?.sending) + const recentEvents = Array.isArray(snapshot.chat?.recentEvents) ? snapshot.chat.recentEvents as Array> : [] + const sawBeforeSend = recentEvents.some(event => String(event?.type || '') === 'before-send') + + addTimeline('chat-submit-poll', { + sending, + messageCount, + sawBeforeSend, + }) + + return sending || sawBeforeSend || messageCount > baselineMessageCount + ? snapshot + : undefined + }, 20_000, 500) + + const finalSnapshot = await waitFor('chat completion after terminal self-acquire turn', async () => { + const snapshot = await callAiriDebugBridge>(chatTargetClient!, 'getSnapshot') + report.debugSnapshots.push(snapshot) + + const sending = Boolean(snapshot.chat?.sending) + const outputText = String(snapshot.chat?.lastTurnComplete?.outputText || '') + const lastMessageRole = String(snapshot.chat?.lastMessage?.role || '') + const lastMessageText = String(snapshot.chat?.lastMessage?.text || '') + const completed = hasCompletedChatTurn(snapshot) + const messageCount = Number(snapshot.chat?.messageCount || 0) + + addTimeline('chat-completion-poll', { + sending, + completed, + messageCount, + lastMessageRole, + outputPreview: summarizeMessageText(outputText), + lastMessagePreview: summarizeMessageText(lastMessageText), + }) + + if (!sending && lastMessageRole === 'error') { + return snapshot + } + + if (!sending && completed && messageCount > baselineMessageCount) { + return snapshot + } + + return undefined + }, 240_000, 1_000) + + const finalStateResult = await callAiriDebugBridge>(chatTargetClient, 'callMcpTool', [{ + name: 'computer_use::desktop_get_state', + arguments: {}, + }]) + report.internalMcp.desktopState = requireStructuredContent(finalStateResult, 'computer_use::desktop_get_state (final)') + + const finalTraceResult = await callAiriDebugBridge>(chatTargetClient, 'callMcpTool', [{ + name: 'computer_use::desktop_get_session_trace', + arguments: { limit: 200 }, + }]) + report.internalMcp.sessionTrace = requireStructuredContent(finalTraceResult, 'computer_use::desktop_get_session_trace (final)') + + report.final = { + providerConfigured: Boolean(finalSnapshot.provider?.configured), + providerId: String(finalSnapshot.provider?.activeProvider || ''), + modelId: String(finalSnapshot.provider?.activeModel || ''), + messageCount: Number(finalSnapshot.chat?.messageCount || 0), + lastMessageRole: String(finalSnapshot.chat?.lastMessage?.role || ''), + lastMessageText: summarizeMessageText(finalSnapshot.chat?.lastMessage?.text), + lastTurnOutput: summarizeMessageText(finalSnapshot.chat?.lastTurnComplete?.outputText), + } + + if (report.final.lastMessageRole === 'error') { + throw new Error(`AIRI chat failed on ${report.final.providerId}/${report.final.modelId}: ${report.final.lastMessageText || 'unknown error'}`) + } + + const finalRunState = ((report.internalMcp.desktopState as Record).runState || {}) as Record + const recentSurfaceDecision = (finalRunState.recentSurfaceDecision || {}) as Record + const stepBindings = Array.isArray(finalRunState.workflowStepTerminalBindings) + ? finalRunState.workflowStepTerminalBindings as Array> + : [] + const ptySessions = Array.isArray(finalRunState.ptySessions) + ? finalRunState.ptySessions as Array> + : [] + const ptyAuditLog = Array.isArray(finalRunState.ptyAuditLog) + ? finalRunState.ptyAuditLog as Array> + : [] + const auditDelta = ptyAuditLog.slice(baselineAuditCount) + const traceEntries = Array.isArray((report.internalMcp.sessionTrace as Record).trace) + ? (report.internalMcp.sessionTrace as Record).trace as Array> + : [] + const newTraceEntries = traceEntries.slice(baselineTraceCount) + const ptyBinding = stepBindings.find(binding => binding.surface === 'pty' && typeof binding.ptySessionId === 'string') + const ptySessionId = String(ptyBinding?.ptySessionId || '') + + assert( + recentSurfaceDecision.surface === 'pty', + `recentSurfaceDecision.surface must be pty, got ${String(recentSurfaceDecision.surface)}`, + ) + assert( + ptySessionId.length > 0, + 'workflowStepTerminalBindings must contain a PTY binding from workflow self-acquire', + ) + assert( + traceHasTerminalExecCommand(newTraceEntries, 'pwd'), + 'session trace must show terminal_exec for pwd before PTY self-acquire', + ) + assert( + traceHasTerminalExecCommand(newTraceEntries, repoChangesCommand), + 'session trace must show terminal_exec for git diff before PTY self-acquire', + ) + assert( + !traceHasTerminalExecCommand(newTraceEntries, 'vim --version'), + 'session trace must not show terminal_exec for vim --version once workflow self-acquires PTY', + ) + assert( + ptySessions.length > baselinePtySessionCount, + `run-state must show a newly created PTY session, baseline=${baselinePtySessionCount}, current=${ptySessions.length}`, + ) + assert( + auditDelta.some(entry => entry.event === 'create'), + 'PTY audit delta must include create from workflow self-acquire', + ) + assert( + auditDelta.some(entry => entry.event === 'read_screen'), + 'PTY audit delta must include read_screen', + ) + assert( + auditDelta.some(entry => entry.event === 'send_input' && String(entry.inputPreview || '').includes('vim --version')), + 'PTY audit delta must include send_input for vim --version', + ) + + const ptyReadResult = await callAiriDebugBridge>(chatTargetClient, 'callMcpTool', [{ + name: 'computer_use::pty_read_screen', + arguments: { sessionId: ptySessionId }, + }]) + const ptyReadData = requireStructuredContent(ptyReadResult, 'computer_use::pty_read_screen (verify)') + const screenContent = String(ptyReadData.screenContent || extractTextContent(ptyReadResult)) + assert( + ptyReadData.status === 'ok' && screenContent.trim().length > 0, + 'final PTY screen must remain readable after workflow self-acquire execution', + ) + + addTimeline('terminal-self-acquire-verified', { + ptySessionId, + recentSurfaceDecision, + auditDeltaCount: auditDelta.length, + newTraceCount: newTraceEntries.length, + }) + + screenshotPaths.push(await captureChatScreenshot(chatTargetClient, 'post-workflow-self-acquire-turn')) + + await callAiriDebugBridge(chatTargetClient, 'clearEvents') + const summaryPrompt = [ + 'Do not call any more tools.', + 'Reply in plain text only.', + 'Summarize this demo in exactly 4 short bullet points for a management audience.', + 'Mention that the workflow started on exec, self-acquired PTY for the interactive validation command, and then completed successfully.', + `Mention the PTY session id ${ptySessionId}.`, + 'Mention that the PTY session remained readable after the workflow completed.', + `End with EXACTLY: ${summaryMarker}`, + ].join('\n') + const summaryBaselineMessageCount = Number(finalSnapshot.chat?.messageCount || 0) + + await callAiriDebugBridge(chatTargetClient, 'sendChatPrompt', [summaryPrompt]) + addTimeline('demo-summary-send-dispatched', { + baselineMessageCount: summaryBaselineMessageCount, + summaryMarker, + }) + + await waitFor('demo summary submission', async () => { + const snapshot = await callAiriDebugBridge>(chatTargetClient!, 'getSnapshot') + report.debugSnapshots.push(snapshot) + + const messageCount = Number(snapshot.chat?.messageCount || 0) + const sending = Boolean(snapshot.chat?.sending) + const recentEvents = Array.isArray(snapshot.chat?.recentEvents) ? snapshot.chat.recentEvents as Array> : [] + const sawBeforeSend = recentEvents.some(event => String(event?.type || '') === 'before-send') + + addTimeline('demo-summary-submit-poll', { + sending, + messageCount, + sawBeforeSend, + }) + + return sending || sawBeforeSend || messageCount > summaryBaselineMessageCount + ? snapshot + : undefined + }, 20_000, 500) + + const demoSummarySnapshot = await waitFor('demo summary completion', async () => { + const snapshot = await callAiriDebugBridge>(chatTargetClient!, 'getSnapshot') + report.debugSnapshots.push(snapshot) + + const sending = Boolean(snapshot.chat?.sending) + const outputText = String(snapshot.chat?.lastTurnComplete?.outputText || '') + const lastMessageRole = String(snapshot.chat?.lastMessage?.role || '') + const lastMessageText = String(snapshot.chat?.lastMessage?.text || '') + const messageCount = Number(snapshot.chat?.messageCount || 0) + const completed = hasCompletedChatTurn(snapshot) + + addTimeline('demo-summary-completion-poll', { + sending, + completed, + messageCount, + lastMessageRole, + outputPreview: summarizeMessageText(outputText), + lastMessagePreview: summarizeMessageText(lastMessageText), + }) + + if (!sending && lastMessageRole === 'error') { + return snapshot + } + + if (!sending && completed && messageCount > summaryBaselineMessageCount && lastMessageRole === 'assistant') { + return snapshot + } + + return undefined + }, 120_000, 1_000) + + const chatMessages = await callAiriDebugBridge>>(chatTargetClient, 'getChatMessages', [6]) + const latestAssistantSummary = [...chatMessages] + .reverse() + .find(message => String(message.role || '') === 'assistant' && String(message.text || '').includes(summaryMarker)) + const demoSummaryText = String( + latestAssistantSummary?.text + || demoSummarySnapshot.chat?.lastMessage?.text + || demoSummarySnapshot.chat?.lastTurnComplete?.outputText + || '', + ).trim() + assert( + demoSummaryText.includes(summaryMarker), + `demo summary text must include "${summaryMarker}"`, + ) + + screenshotPaths.push(await captureChatScreenshot(chatTargetClient, 'demo-summary')) + + report.final = { + providerConfigured: Boolean(demoSummarySnapshot.provider?.configured), + providerId: String(demoSummarySnapshot.provider?.activeProvider || report.final?.providerId || ''), + modelId: String(demoSummarySnapshot.provider?.activeModel || report.final?.modelId || ''), + messageCount: Number(demoSummarySnapshot.chat?.messageCount || report.final?.messageCount || 0), + lastMessageRole: String(demoSummarySnapshot.chat?.lastMessage?.role || report.final?.lastMessageRole || ''), + lastMessageText: summarizeMessageText(demoSummarySnapshot.chat?.lastMessage?.text || report.final?.lastMessageText), + lastTurnOutput: summarizeMessageText(demoSummarySnapshot.chat?.lastTurnComplete?.outputText || report.final?.lastTurnOutput), + ptySessionId, + demoSummaryText, + screenshotPaths: [...screenshotPaths], + } + + await writeDemoSummary({ + providerId: String(report.final.providerId || ''), + modelId: String(report.final.modelId || ''), + ptySessionId, + recentSurfaceDecision, + auditDeltaCount: auditDelta.length, + newTraceCount: newTraceEntries.length, + screenContent, + demoSummaryText, + screenshotPaths, + }) + + await callAiriDebugBridge(chatTargetClient, 'callMcpTool', [{ + name: 'computer_use::pty_destroy', + arguments: { sessionId: ptySessionId }, + }]).catch(() => undefined) + + report.status = 'completed' + await writeReport() + + console.info(JSON.stringify({ + ok: true, + reportPath, + demoSummaryPath, + providerId: report.final.providerId, + modelId: report.final.modelId, + ptySessionId, + demoSummaryText: report.final.demoSummaryText, + screenshotPaths: report.final.screenshotPaths, + lastTurnOutput: report.final.lastTurnOutput, + lastMessageText: report.final.lastMessageText, + }, null, 2)) + } + catch (error) { + report.status = 'failed' + report.error = error instanceof Error ? error.stack || error.message : String(error) + addTimeline('failure', { error: report.error }) + await writeReport() + console.error(report.error) + exitCode = 1 + } + finally { + if (chatTargetClient && !chatClientSharesMainTarget) { + await chatTargetClient.close().catch(() => {}) + } + await mainTargetClient?.close().catch(() => {}) + + if (stageProcess && !stageProcess.killed) { + stageProcess.kill('SIGINT') + await sleep(1_500) + if (stageProcess.exitCode == null) { + stageProcess.kill('SIGTERM') + } + } + + await writeReport().catch(() => {}) + } +} + +main().finally(() => { + exit(exitCode) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-airi-discord-agentic.ts b/services/computer-use-mcp/src/bin/e2e-airi-discord-agentic.ts new file mode 100644 index 000000000..cc7d4a4d2 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-airi-discord-agentic.ts @@ -0,0 +1,1326 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' + +import type { AiriDebugSnapshotLike } from '../e2e/debug-targets' + +import { execFile, spawn } from 'node:child_process' +import { createWriteStream } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +import WebSocket from 'ws' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +import { hasCompletedChatTurn } from '../e2e/chat-turn' +import { + isChatSurfaceTarget, + prioritizeInspectableAiriTargets, +} from '../e2e/debug-targets' +import { getProviderBootstrapConfig } from '../e2e/provider-bootstrap' + +interface DebugTarget { + id: string + title: string + type: string + url: string + webSocketDebuggerUrl?: string +} + +interface TimelineEntry { + at: string + event: string + detail?: Record +} + +interface DiscordBotRuntimeState { + attemptedConnect: boolean + connected: boolean + receivedConfig: boolean + readyUserTag?: string + waitingForConfiguration: boolean + applyFailure?: string +} + +interface ReportShape { + startedAt: string + finishedAt?: string + status: 'running' | 'completed' | 'failed' + scenario: 'discord-agentic' + prompt: string + reportDir: string + paths: { + reportPath: string + stageLogPath: string + discordBotLogPath: string + mcpSessionRoot: string + auditLogPath?: string + screenshotsDir?: string + } + timeline: TimelineEntry[] + debugSnapshots: unknown[] + mcp: { + capabilities?: unknown + desktopState?: unknown + sessionTrace?: unknown + } + discord: { + allowLoginFailure: boolean + expectedTokenLength: number + tokenSource: 'auto' | 'portal' | 'local' + providerServerUrl: string + ui?: { + route?: string + enabled?: boolean + configured?: boolean + tokenLength?: number + } + bot?: DiscordBotRuntimeState + } + final?: { + providerConfigured?: boolean + providerId?: string + modelId?: string + messageCount?: number + lastMessageRole?: string + lastMessageText?: string + lastTurnOutput?: string + toolCallCount?: number + toolResultCount?: number + } + error?: string +} + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const repoDir = resolve(packageDir, '../..') +const preferredDebugPort = Number(env.AIRI_E2E_DEBUG_PORT || '9222') +const runId = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-') +const preferredProviderId = env.AIRI_E2E_PROVIDER?.trim() || 'github-models' +const preferredModelCandidates = Array.from(new Set( + (env.AIRI_E2E_MODELS?.trim() + ? env.AIRI_E2E_MODELS.split(',') + : [env.AIRI_E2E_MODEL?.trim() || 'openai/gpt-4o-mini', 'openai/gpt-4.1-mini']) + .map(model => model?.trim()) + .filter((model): model is string => Boolean(model)), +)) +const discordTokenSource = (['auto', 'portal', 'local'].includes((env.AIRI_E2E_DISCORD_TOKEN_SOURCE || '').trim().toLowerCase()) + ? (env.AIRI_E2E_DISCORD_TOKEN_SOURCE || '').trim().toLowerCase() + : 'auto') as 'auto' | 'portal' | 'local' +const promptMarker = `airi-discord-agentic-${runId.slice(-8)}` +// NOTICE: Keep this ASCII-only so Quartz typing does not interact badly with IME composition. +const defaultPromptByTokenSource: Record = { + auto: 'Configure AIRI Discord automatically. Do not ask the human for the token. Prefer retrieving the token from the live Discord browser or Developer Portal session by using browser_dom_* tools, browser_agent_run, clipboard_read_text, clipboard_write_text, and desktop/browser control. If no live token can be retrieved, fall back to local sources such as .env. Then open AIRI Discord settings, enable the module, fill the token, save it, and verify the module is configured. Keep going until verified, not narrated.', + portal: 'Configure AIRI Discord automatically. Do not ask the human for the token and do not use .env as the primary source. Use browser_dom_* tools, browser_agent_run, clipboard_read_text, clipboard_write_text, and desktop/browser control to retrieve the token from the live Discord browser or Developer Portal session, then open AIRI Discord settings, enable the module, fill the token, save it, and verify the module is configured. Keep going until verified, not narrated.', + local: 'Configure AIRI Discord automatically. Do not ask the human for the token. Use available computer-use MCP tools, especially terminal plus desktop or browser tools, to retrieve the local Discord token from .env by yourself, open AIRI Discord settings, enable the module, fill the token, save it, and verify the module is configured. If the only local token is a placeholder or test token, still use it for this testing run. Keep going until verified, not narrated.', +} +const promptBaseText = env.AIRI_E2E_DISCORD_AGENTIC_PROMPT?.trim() + || defaultPromptByTokenSource[discordTokenSource] +const promptText = `${promptBaseText} [${promptMarker}]` +const reportDir = resolve(packageDir, '.computer-use-mcp', 'reports', `airi-discord-agentic-${runId}`) +const reportPath = resolve(reportDir, 'report.json') +const stageLogPath = resolve(reportDir, 'stage-tamagotchi.log') +const discordBotLogPath = resolve(reportDir, 'discord-bot.log') +const mcpSessionRoot = resolve(reportDir, 'computer-use-session') +const rootEnvPath = resolve(repoDir, '.env') +const verifyDiscordBot = parseBooleanEnv(env.AIRI_E2E_DISCORD_VERIFY_BOT, false) + +const execFileAsync = promisify(execFile) + +const report: ReportShape = { + startedAt: new Date().toISOString(), + status: 'running', + scenario: 'discord-agentic', + prompt: promptText, + reportDir, + paths: { + reportPath, + stageLogPath, + discordBotLogPath, + mcpSessionRoot, + }, + timeline: [], + debugSnapshots: [], + mcp: {}, + discord: { + allowLoginFailure: false, + expectedTokenLength: 0, + tokenSource: discordTokenSource, + providerServerUrl: env.AIRI_URL || 'ws://localhost:6121/ws', + }, +} + +function addTimeline(event: string, detail?: Record) { + report.timeline.push({ + at: new Date().toISOString(), + event, + detail, + }) +} + +function pushSnapshot(source: string, snapshot: Record) { + report.debugSnapshots.push({ + at: new Date().toISOString(), + source, + snapshot, + }) +} + +function parseBooleanEnv(value: string | undefined, fallback = false) { + if (!value?.trim()) { + return fallback + } + + return ['1', 'true', 'yes', 'on'].includes(value.trim().toLowerCase()) +} + +function parseCommandArgs(raw: string | undefined, fallback: string[]) { + if (!raw?.trim()) { + return fallback + } + + return raw + .split(/\s+/) + .map(item => item.trim()) + .filter(Boolean) +} + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') { + throw new Error(`${label} did not return an object result`) + } + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') { + throw new Error(`${label} missing structuredContent`) + } + + return structuredContent as Record +} + +function sleep(ms: number) { + return new Promise(resolvePromise => setTimeout(resolvePromise, ms)) +} + +async function withTimeout(label: string, task: Promise, timeoutMs: number) { + let timeoutHandle: NodeJS.Timeout | undefined + + try { + return await Promise.race([ + task, + new Promise((_resolvePromise, rejectPromise) => { + timeoutHandle = setTimeout(() => rejectPromise(new Error(`Timed out waiting for ${label}`)), timeoutMs) + }), + ]) + } + finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle) + } + } +} + +async function writeReport() { + report.finishedAt = new Date().toISOString() + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf-8') +} + +async function canListenOnPort(port: number) { + return await new Promise((resolvePromise) => { + const server = createServer() + server.once('error', () => { + resolvePromise(false) + }) + server.listen(port, '127.0.0.1', () => { + server.close(() => resolvePromise(true)) + }) + }) +} + +async function findAvailablePort(preferredPort: number, attempts = 20) { + for (let index = 0; index < attempts; index += 1) { + const candidate = preferredPort + index + if (await canListenOnPort(candidate)) { + return candidate + } + } + + throw new Error(`Could not find an available remote debug port starting from ${preferredPort}`) +} + +async function terminateExistingStageTamagotchiInstances() { + const patterns = [ + resolve(repoDir, 'apps', 'stage-tamagotchi'), + '@proj-airi/stage-tamagotchi', + resolve(repoDir, 'node_modules', '.pnpm', 'electron@'), + ] + + for (const pattern of patterns) { + await execFileAsync('pkill', ['-f', pattern]).catch(() => {}) + } + + await sleep(1_500) +} + +async function terminateExistingDiscordBotInstances() { + const patterns = [ + resolve(repoDir, 'services', 'discord-bot'), + '@proj-airi/discord-bot', + ] + + for (const pattern of patterns) { + await execFileAsync('pkill', ['-f', pattern]).catch(() => {}) + } + + await sleep(1_000) +} + +async function waitFor(label: string, task: () => Promise, timeoutMs = 60_000, intervalMs = 500) { + const startedAt = Date.now() + + while ((Date.now() - startedAt) < timeoutMs) { + const value = await task() + if (value !== undefined) { + return value + } + + await sleep(intervalMs) + } + + throw new Error(`Timed out waiting for ${label}`) +} + +function parseDotEnv(text: string) { + const values: Record = {} + + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#')) { + continue + } + + const separatorIndex = trimmed.indexOf('=') + if (separatorIndex <= 0) { + continue + } + + const key = trimmed.slice(0, separatorIndex).trim() + const rawValue = trimmed.slice(separatorIndex + 1).trim() + const unwrapped = rawValue.replace(/^['"]|['"]$/gu, '') + values[key] = unwrapped + } + + return values +} + +async function readRootEnvValues() { + try { + const raw = await readFile(rootEnvPath, 'utf-8') + return parseDotEnv(raw) + } + catch { + return {} + } +} + +function resolveConfigValue(name: string, fallbackValues: Record) { + const processValue = env[name]?.trim() + if (processValue) { + return processValue + } + + const fileValue = fallbackValues[name]?.trim() + if (fileValue) { + return fileValue + } + + return '' +} + +function looksLikePlaceholderSecret(value: string) { + const normalized = value.trim().toLowerCase() + if (!normalized) { + return true + } + + return normalized.includes('replace') + || normalized.includes('placeholder') + || normalized.includes('example') + || normalized.includes('your-') + || normalized === 'changeme' +} + +function createLineListener(onLine: (line: string) => void) { + let buffer = '' + + return (chunk: { toString: (encoding: string) => string }) => { + buffer += chunk.toString('utf-8') + const lines = buffer.split(/\r?\n/u) + buffer = lines.pop() ?? '' + + for (const line of lines) { + const trimmed = line.trim() + if (trimmed) { + onLine(trimmed) + } + } + } +} + +class CdpClient { + private ws: any + private nextId = 0 + private pending = new Map void, reject: (error: Error) => void }>() + + static async connectToUrl(webSocketUrl: string, options: { enableRuntime?: boolean, enablePage?: boolean } = {}) { + const client = new CdpClient() + client.ws = new WebSocket(webSocketUrl) + + await new Promise((resolvePromise, rejectPromise) => { + const onOpen = () => resolvePromise() + const onError = (error: Error) => rejectPromise(error) + + client.ws.addEventListener('open', onOpen, { once: true }) + client.ws.addEventListener('error', onError, { once: true }) + }) + + client.ws.addEventListener('message', (event: { data: string }) => { + const payload = JSON.parse(event.data) + if (typeof payload.id === 'number') { + const pending = client.pending.get(payload.id) + if (!pending) { + return + } + + client.pending.delete(payload.id) + if (payload.error) { + pending.reject(new Error(String(payload.error.message || 'Unknown CDP error'))) + return + } + + pending.resolve(payload.result) + } + }) + + if (options.enableRuntime !== false) { + await client.send('Runtime.enable') + } + + if (options.enablePage !== false) { + await client.send('Page.enable') + } + + return client + } + + static async connect(target: DebugTarget) { + if (!target.webSocketDebuggerUrl) { + throw new Error(`Debug target ${target.title || target.id} does not expose webSocketDebuggerUrl`) + } + + return await CdpClient.connectToUrl(target.webSocketDebuggerUrl) + } + + async send(method: string, params?: Record) { + const id = ++this.nextId + const payload = { id, method, params } + + return await new Promise((resolvePromise, rejectPromise) => { + this.pending.set(id, { resolve: resolvePromise, reject: rejectPromise }) + this.ws.send(JSON.stringify(payload)) + }) + } + + async evaluate(expression: string): Promise { + const result = await this.send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + userGesture: true, + }) + + if (result?.exceptionDetails) { + const text = result.exceptionDetails.text || 'Runtime.evaluate exception' + throw new Error(String(text)) + } + + return result?.result?.value as T + } + + async close() { + if (this.ws?.readyState === 1) { + this.ws.close() + } + } +} + +async function listDebugTargets(browserWsUrl: string) { + const browserClient = await CdpClient.connectToUrl(browserWsUrl, { + enableRuntime: false, + enablePage: false, + }) + + try { + const result = await browserClient.send('Target.getTargets') as { targetInfos?: Array> } + const targetInfos = Array.isArray(result.targetInfos) ? result.targetInfos : [] + + return targetInfos + .filter(target => target.type === 'page') + .map((target) => { + const targetId = String(target.targetId || '') + return { + id: targetId, + title: String(target.title || ''), + type: String(target.type || ''), + url: String(target.url || ''), + webSocketDebuggerUrl: browserWsUrl.replace(/\/devtools\/browser\/[^/]+$/, `/devtools/page/${targetId}`), + } satisfies DebugTarget + }) + } + finally { + await browserClient.close().catch(() => {}) + } +} + +async function bringTargetToFront(client: CdpClient, label: string) { + await client.send('Page.bringToFront') + addTimeline('target-brought-to-front', { label }) + await sleep(750) +} + +async function getAiriDebugSnapshot(client: CdpClient) { + return await client.evaluate(`(() => { + const bridge = window.__AIRI_DEBUG__ + if (!bridge || typeof bridge.getSnapshot !== 'function') { + return undefined + } + + return bridge.getSnapshot() + })()`) +} + +async function waitForChatSurfaceReady(client: CdpClient, label: string) { + return await waitFor(label, async () => { + try { + const snapshot = await client.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + if (snapshot.dom?.hasTextarea) { + return snapshot + } + + return undefined + } + catch { + return undefined + } + }, 30_000, 250) +} + +async function findTargetWithAiriDebugBridge( + browserWsUrl: string, + label: string, + predicate?: (target: DebugTarget, snapshot: AiriDebugSnapshotLike) => boolean, +) { + return await waitFor(label, async () => { + const targets = prioritizeInspectableAiriTargets(await listDebugTargets(browserWsUrl).catch(() => [])) + + for (const target of targets) { + let client: CdpClient | undefined + + try { + client = await withTimeout( + `${label} connect ${target.title || target.url || target.id}`, + CdpClient.connect(target), + 2_500, + ) + const snapshot = await withTimeout( + `${label} snapshot ${target.title || target.url || target.id}`, + getAiriDebugSnapshot(client), + 2_500, + ) + if (!snapshot) { + continue + } + + if (predicate && !predicate(target, snapshot)) { + continue + } + + return { + target, + snapshot, + } + } + catch { + continue + } + finally { + await client?.close().catch(() => {}) + } + } + + return undefined + }, 90_000, 750) +} + +function summarizeMessageText(value: unknown) { + if (typeof value !== 'string') { + return '' + } + + const normalized = value.replace(/\s+/g, ' ').trim() + return normalized.length > 240 ? `${normalized.slice(0, 237)}...` : normalized +} + +let exitCode = 0 + +async function main() { + let stageProcess: ChildProcessWithoutNullStreams | undefined + let discordBotProcess: ChildProcessWithoutNullStreams | undefined + let mcpClient: Client | undefined + let mainTargetClient: CdpClient | undefined + let chatTargetClient: CdpClient | undefined + let chatClientSharesMainTarget = false + let chatSurfaceMode: 'separate-window' | 'same-window-route' = 'separate-window' + let browserWsUrl: string | undefined + const debugPort = await findAvailablePort(preferredDebugPort) + const rootEnvValues = await readRootEnvValues() + const providerBootstrapConfig = getProviderBootstrapConfig({ + providerId: preferredProviderId, + processEnv: env, + dotenvValues: rootEnvValues, + }) + const allowLoginFailure = parseBooleanEnv(resolveConfigValue('AIRI_E2E_DISCORD_ALLOW_LOGIN_FAILURE', rootEnvValues), false) + const discordToken = resolveConfigValue('AIRI_E2E_DISCORD_TOKEN', rootEnvValues) + || resolveConfigValue('DISCORD_TOKEN', rootEnvValues) + const hasLocalDiscordToken = Boolean(discordToken.trim()) + const discordRuntimeState: DiscordBotRuntimeState = { + attemptedConnect: false, + connected: false, + receivedConfig: false, + waitingForConfiguration: false, + } + + report.discord.allowLoginFailure = allowLoginFailure + report.discord.expectedTokenLength = discordToken.length + + try { + await mkdir(reportDir, { recursive: true }) + await mkdir(mcpSessionRoot, { recursive: true }) + + if (discordTokenSource === 'local' && !hasLocalDiscordToken) { + throw new Error(`Discord agentic demo with AIRI_E2E_DISCORD_TOKEN_SOURCE=local requires AIRI_E2E_DISCORD_TOKEN (or DISCORD_TOKEN) in process env or ${rootEnvPath}. No local token-like value is available for AIRI to retrieve.`) + } + + if (hasLocalDiscordToken && looksLikePlaceholderSecret(discordToken)) { + addTimeline('discord-token-placeholder-mode', { + expectedTokenLength: discordToken.length, + verifyDiscordBot, + allowLoginFailure, + }) + + if (verifyDiscordBot && !allowLoginFailure) { + throw new Error('Discord bot verification requires a real token or AIRI_E2E_DISCORD_ALLOW_LOGIN_FAILURE=true. The current local token value is only a placeholder.') + } + } + + addTimeline('bootstrap', { + reportDir, + debugPort, + allowLoginFailure, + verifyDiscordBot, + tokenSource: discordTokenSource, + hasLocalDiscordToken, + }) + await terminateExistingStageTamagotchiInstances() + if (verifyDiscordBot) { + await terminateExistingDiscordBotInstances() + } + addTimeline('terminated-stale-processes', { stage: true, discordBot: verifyDiscordBot }) + + const stageLogStream = createWriteStream(stageLogPath, { flags: 'a' }) + const discordBotLogStream = verifyDiscordBot + ? createWriteStream(discordBotLogPath, { flags: 'a' }) + : undefined + + addTimeline('start-stage-tamagotchi') + stageProcess = spawn('pnpm', ['-F', '@proj-airi/stage-tamagotchi', 'dev'], { + cwd: repoDir, + env: { + ...env, + APP_REMOTE_DEBUG: 'true', + APP_REMOTE_DEBUG_PORT: String(debugPort), + APP_REMOTE_DEBUG_NO_OPEN: 'true', + }, + stdio: 'pipe', + }) + + const onStageChunk = createLineListener((line) => { + const match = line.match(/DevTools listening on (ws:\/\/\S+)/) + if (match?.[1]) { + browserWsUrl = match[1] + } + }) + + stageProcess.stdout.on('data', (chunk) => { + stageLogStream.write(chunk) + onStageChunk(chunk) + }) + stageProcess.stderr.on('data', (chunk) => { + stageLogStream.write(chunk) + onStageChunk(chunk) + }) + + stageProcess.on('exit', (code, signal) => { + addTimeline('stage-tamagotchi-exit', { + code: code ?? undefined, + signal: signal ?? undefined, + }) + }) + + const activeBrowserWsUrl = await waitFor('remote debug browser websocket', async () => { + return browserWsUrl + }, 120_000, 500) + addTimeline('remote-debug-browser-ready', { browserWsUrl: activeBrowserWsUrl, debugPort }) + + const mainTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'AIRI main target', + (_target, snapshot) => !String(snapshot.route || '').includes('/chat'), + ) + const mainTarget = mainTargetMatch.target + pushSnapshot('main-target-initial', mainTargetMatch.snapshot as unknown as Record) + addTimeline('main-target-ready', { + title: mainTarget.title, + url: mainTarget.url, + route: mainTargetMatch.snapshot.route, + documentTitle: mainTargetMatch.snapshot.documentTitle, + }) + + mainTargetClient = await CdpClient.connect(mainTarget) + await bringTargetToFront(mainTargetClient, 'main') + + if (verifyDiscordBot) { + addTimeline('start-discord-bot') + discordBotProcess = spawn('pnpm', ['-F', '@proj-airi/discord-bot', 'start'], { + cwd: repoDir, + env: { + ...env, + DISCORD_TOKEN: '', + AIRI_TOKEN: env.AIRI_TOKEN || 'abcd', + AIRI_URL: report.discord.providerServerUrl, + }, + stdio: 'pipe', + }) + + const onDiscordBotChunk = createLineListener((line) => { + if (line.includes('Waiting for configuration from UI')) { + discordRuntimeState.waitingForConfiguration = true + addTimeline('discord-bot-waiting-for-ui-config') + } + if (line.includes('Received Discord configuration:')) { + discordRuntimeState.receivedConfig = true + addTimeline('discord-bot-received-config') + } + if (line.includes('Connecting Discord client...')) { + discordRuntimeState.attemptedConnect = true + addTimeline('discord-bot-connecting') + } + if (line.includes('Discord client connected.')) { + discordRuntimeState.connected = true + addTimeline('discord-bot-connected') + } + if (line.includes('Discord bot ready! User:')) { + discordRuntimeState.connected = true + discordRuntimeState.readyUserTag = line.split('Discord bot ready! User:').at(1)?.trim() || undefined + addTimeline('discord-bot-ready', { + userTag: discordRuntimeState.readyUserTag, + }) + } + if (line.includes('Failed to apply Discord configuration.')) { + discordRuntimeState.applyFailure = line + addTimeline('discord-bot-apply-failure', { line }) + } + }) + + discordBotProcess.stdout.on('data', (chunk) => { + discordBotLogStream?.write(chunk) + onDiscordBotChunk(chunk) + }) + discordBotProcess.stderr.on('data', (chunk) => { + discordBotLogStream?.write(chunk) + onDiscordBotChunk(chunk) + }) + + discordBotProcess.on('exit', (code, signal) => { + addTimeline('discord-bot-exit', { + code: code ?? undefined, + signal: signal ?? undefined, + }) + }) + + await waitFor('discord bot startup', async () => { + return discordRuntimeState.waitingForConfiguration ? true : undefined + }, 45_000, 500) + } + else { + addTimeline('discord-bot-verification-skipped') + } + + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start']) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'macos-local', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_OPENABLE_APPS: 'Terminal,Cursor,Google Chrome,Electron,Discord', + COMPUTER_USE_DENY_APPS: '1Password,Keychain,System Settings,Activity Monitor', + COMPUTER_USE_BROWSER_DOM_BRIDGE_ENABLED: 'true', + COMPUTER_USE_BROWSER_DOM_BRIDGE_HOST: env.COMPUTER_USE_BROWSER_DOM_BRIDGE_HOST || '127.0.0.1', + COMPUTER_USE_BROWSER_DOM_BRIDGE_PORT: env.COMPUTER_USE_BROWSER_DOM_BRIDGE_PORT || '8765', + COMPUTER_USE_SESSION_TAG: `airi-discord-agentic-${runId}`, + COMPUTER_USE_ALLOWED_BOUNDS: env.COMPUTER_USE_ALLOWED_BOUNDS || '0,0,2560,1600', + COMPUTER_USE_SESSION_ROOT: mcpSessionRoot, + }, + stderr: 'pipe', + }) + + mcpClient = new Client({ + name: '@proj-airi/computer-use-mcp-e2e-airi-discord-agentic', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk: { toString: (encoding: string) => string }) => { + const text = chunk.toString('utf-8').trim() + if (text) { + addTimeline('computer-use-mcp-stderr', { text }) + } + }) + + await mcpClient.connect(transport) + addTimeline('computer-use-mcp-connected') + + const capabilities = await mcpClient.callTool({ + name: 'desktop_get_capabilities', + arguments: {}, + }) + const capabilitiesData = requireStructuredContent(capabilities, 'desktop_get_capabilities') + report.mcp.capabilities = capabilitiesData + report.paths.auditLogPath = String((capabilitiesData.session as Record | undefined)?.auditLogPath || '') || undefined + report.paths.screenshotsDir = String((capabilitiesData.session as Record | undefined)?.screenshotsDir || '') || undefined + addTimeline('desktop-capabilities', { + executionMode: (capabilitiesData.executionTarget as Record | undefined)?.mode, + auditLogPath: report.paths.auditLogPath, + screenshotsDir: report.paths.screenshotsDir, + browserAgentReady: Boolean((capabilitiesData.browserAgent as Record | undefined)?.rootExists), + }) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'before-open-chat' }, + }) + addTimeline('screenshot-captured', { label: 'before-open-chat' }) + + try { + await withTimeout( + 'AIRI debug bridge openChat', + mainTargetClient.evaluate('window.__AIRI_DEBUG__.openChat()'), + 8_000, + ) + addTimeline('chat-open-requested', { mode: 'separate-window' }) + + const chatTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'Chat target', + (target, snapshot) => isChatSurfaceTarget(target, snapshot), + ) + const chatTarget = chatTargetMatch.target + pushSnapshot('chat-target-initial', chatTargetMatch.snapshot as unknown as Record) + addTimeline('chat-target-ready', { + title: chatTarget.title, + url: chatTarget.url, + route: chatTargetMatch.snapshot.route, + documentTitle: chatTargetMatch.snapshot.documentTitle, + mode: 'separate-window', + }) + + chatTargetClient = await CdpClient.connect(chatTarget) + await bringTargetToFront(chatTargetClient, 'chat') + const readyChatSnapshot = await waitForChatSurfaceReady(chatTargetClient, 'chat surface ready') + pushSnapshot('chat-surface-ready', readyChatSnapshot) + addTimeline('chat-surface-ready', { + route: String(readyChatSnapshot.route || ''), + hasTextarea: Boolean(readyChatSnapshot.dom?.hasTextarea), + }) + } + catch (error) { + chatSurfaceMode = 'same-window-route' + addTimeline('chat-open-fallback', { + mode: 'same-window-route', + reason: error instanceof Error ? error.message : String(error), + }) + + await mainTargetClient.evaluate(`window.__AIRI_DEBUG__.navigateTo('/chat')`) + + const fallbackChatSnapshot = await waitFor('chat route in main AIRI window', async () => { + try { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + const onChatRoute = String(snapshot.route || '').includes('/chat') + const hasTextarea = Boolean(snapshot.dom?.hasTextarea) + return onChatRoute && hasTextarea ? snapshot : undefined + } + catch { + return undefined + } + }, 30_000, 750) + pushSnapshot('chat-target-fallback', fallbackChatSnapshot) + + chatTargetClient = mainTargetClient + await bringTargetToFront(chatTargetClient, 'main-chat-fallback') + const readyChatSnapshot = await waitForChatSurfaceReady(chatTargetClient, 'fallback chat surface ready') + pushSnapshot('chat-surface-ready', readyChatSnapshot) + addTimeline('chat-surface-ready', { + route: String(readyChatSnapshot.route || ''), + hasTextarea: Boolean(readyChatSnapshot.dom?.hasTextarea), + mode: 'same-window-route', + }) + chatClientSharesMainTarget = true + addTimeline('chat-target-ready', { + title: 'AIRI', + url: 'http://localhost:5173/#/chat', + mode: 'same-window-route', + }) + } + + const focusedDesktop = await mcpClient.callTool({ + name: 'desktop_focus_app', + arguments: { app: 'Electron' }, + }) + const focusedDesktopData = requireStructuredContent(focusedDesktop, 'desktop_focus_app') + addTimeline('desktop-focus-app', { + app: 'Electron', + status: focusedDesktopData.status, + }) + + const observation = await waitFor('Chat window observation', async () => { + const result = await mcpClient!.callTool({ + name: 'desktop_observe_windows', + arguments: { limit: 24 }, + }) + const data = requireStructuredContent(result, 'desktop_observe_windows') + const observationPayload = ((data.backendResult as Record | undefined)?.observation + || data.observation) as Record | undefined + const windows = Array.isArray(observationPayload?.windows) ? observationPayload.windows as Array> : [] + const frontmostAppName = String(observationPayload?.frontmostAppName || '') + const chatWindow = windows.find(window => String(window.title || '').includes('AIRI')) + if (!frontmostAppName.includes('Electron')) { + return undefined + } + if (!chatWindow) { + return undefined + } + + return { + full: data, + chatWindow, + } + }, 30_000, 1_000) + addTimeline('chat-window-observed', { + ...observation.chatWindow, + mode: chatSurfaceMode, + }) + + await chatTargetClient.evaluate('window.__AIRI_DEBUG__.clearEvents()') + const selectionSnapshot = await chatTargetClient.evaluate>(`window.__AIRI_DEBUG__.ensureConsciousnessSelection(${JSON.stringify({ + provider: preferredProviderId, + preferredModels: preferredModelCandidates, + providerConfig: providerBootstrapConfig, + })})`) + pushSnapshot('consciousness-selection', selectionSnapshot) + addTimeline('consciousness-selection-ready', { + providerId: String(selectionSnapshot.provider?.activeProvider || ''), + modelId: String(selectionSnapshot.provider?.activeModel || ''), + preferredProviderId, + preferredModelCandidates, + providerAvailable: Boolean(selectionSnapshot.provider?.providerAvailable), + providerBootstrapped: Boolean(providerBootstrapConfig), + }) + + if (preferredProviderId === 'github-models' && !selectionSnapshot.provider?.providerAvailable) { + throw new Error(`GitHub Models provider is unavailable before chat send. Checked .env at ${rootEnvPath} for bootstrap credentials, but AIRI still did not validate github-models.`) + } + + await chatTargetClient.evaluate('window.__AIRI_DEBUG__.clearEvents()') + const resetSnapshot = await chatTargetClient.evaluate>('window.__AIRI_DEBUG__.resetChatSession()') + pushSnapshot('chat-reset', resetSnapshot) + addTimeline('chat-session-reset', { + providerConfigured: Boolean(resetSnapshot.provider?.configured), + providerId: String(resetSnapshot.provider?.activeProvider || ''), + modelId: String(resetSnapshot.provider?.activeModel || ''), + messageCount: Number(resetSnapshot.chat?.messageCount || 0), + activeSessionId: String(resetSnapshot.chat?.activeSessionId || ''), + }) + + const focusState = await waitFor('chat textarea focus', async () => { + const state = await chatTargetClient!.evaluate>(`(() => { + window.focus() + const textarea = document.querySelector('textarea.ph-no-capture') + if (!(textarea instanceof HTMLTextAreaElement)) { + return { + ok: false, + reason: 'textarea-not-found', + } + } + + textarea.click() + textarea.focus() + + return { + ok: document.activeElement === textarea, + placeholder: textarea.getAttribute('placeholder'), + valueLength: textarea.value.length, + disabled: textarea.disabled, + readOnly: textarea.readOnly, + focusedTagName: document.activeElement?.tagName || '', + } + })()`) + + addTimeline('textarea-focus-poll', { + ok: Boolean(state.ok), + disabled: Boolean(state.disabled), + readOnly: Boolean(state.readOnly), + focusedTagName: String(state.focusedTagName || ''), + }) + + return state.ok ? state : undefined + }, 15_000, 250) + addTimeline('textarea-focused', { + placeholder: String(focusState.placeholder || ''), + valueLength: Number(focusState.valueLength || 0), + }) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'chat-before-type' }, + }) + addTimeline('screenshot-captured', { label: 'chat-before-type' }) + + const baselineMessageCount = Number(resetSnapshot.chat?.messageCount || 0) + + const typed = await mcpClient.callTool({ + name: 'desktop_type_text', + arguments: { + text: promptText, + pressEnter: false, + captureAfter: true, + }, + }) + const typedData = requireStructuredContent(typed, 'desktop_type_text') + addTimeline('desktop-type-text', { + status: typedData.status, + screenshotPath: (typedData.screenshot as Record | undefined)?.path, + }) + + const typedSnapshot = await waitFor('typed prompt to settle in textarea', async () => { + const typedState = await chatTargetClient!.evaluate>(`(() => { + const textarea = document.querySelector('textarea.ph-no-capture') + const value = textarea instanceof HTMLTextAreaElement ? textarea.value : '' + return { + value, + valueLength: value.length, + containsPromptMarker: value.includes(${JSON.stringify(promptMarker)}), + } + })()`) + + addTimeline('textarea-poll', { + valueLength: Number(typedState.valueLength || 0), + containsPromptMarker: Boolean(typedState.containsPromptMarker), + }) + + return typedState.containsPromptMarker === true ? typedState : undefined + }, 10_000, 250) + addTimeline('textarea-filled', { + valueLength: Number(typedSnapshot.valueLength || 0), + }) + + const submit = await mcpClient.callTool({ + name: 'desktop_press_keys', + arguments: { + keys: ['enter'], + captureAfter: true, + }, + }) + const submitData = requireStructuredContent(submit, 'desktop_press_keys') + addTimeline('desktop-press-keys', { + status: submitData.status, + screenshotPath: (submitData.screenshot as Record | undefined)?.path, + }) + + const submittedSnapshot = await waitFor('chat submit', async () => { + const snapshot = await chatTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + pushSnapshot('chat-submit-poll', snapshot) + + const messageCount = Number(snapshot.chat?.messageCount || 0) + const sending = Boolean(snapshot.chat?.sending) + const lastMessageRole = String(snapshot.chat?.lastMessage?.role || '') + const lastMessageText = String(snapshot.chat?.lastMessage?.text || '') + const recentEvents = Array.isArray(snapshot.chat?.recentEvents) ? snapshot.chat.recentEvents as Array> : [] + const sawBeforeSend = recentEvents.some(event => String(event?.type || '') === 'before-send') + + addTimeline('chat-submit-poll', { + sending, + messageCount, + lastMessageRole, + sawBeforeSend, + textareaValueLength: Number(snapshot.dom?.textareaValueLength || 0), + }) + + if (sending || sawBeforeSend) { + return snapshot + } + + if (messageCount > baselineMessageCount && lastMessageRole === 'user' && lastMessageText.includes(promptMarker)) { + return snapshot + } + + return undefined + }, 15_000, 500) + addTimeline('chat-submit-observed', { + sending: Boolean(submittedSnapshot.chat?.sending), + messageCount: Number(submittedSnapshot.chat?.messageCount || 0), + lastMessageRole: String(submittedSnapshot.chat?.lastMessage?.role || ''), + }) + + let capturedStreamingScreenshot = false + const finalSnapshot = await waitFor('chat completion or error', async () => { + const snapshot = await chatTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + pushSnapshot('chat-completion-poll', snapshot) + + if (!capturedStreamingScreenshot && snapshot.chat?.sending && typeof snapshot.chat?.streamingText === 'string' && snapshot.chat.streamingText.trim().length > 0) { + capturedStreamingScreenshot = true + await mcpClient!.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'chat-during-stream' }, + }) + addTimeline('screenshot-captured', { + label: 'chat-during-stream', + streamingLength: snapshot.chat.streamingText.length, + }) + } + + const messageCount = Number(snapshot.chat?.messageCount || 0) + const sending = Boolean(snapshot.chat?.sending) + const lastMessageRole = String(snapshot.chat?.lastMessage?.role || '') + const hasTurnCompletion = hasCompletedChatTurn(snapshot) + const recentEvents = Array.isArray(snapshot.chat?.recentEvents) + ? snapshot.chat.recentEvents as Array> + : [] + const abortedByUser = recentEvents.some(event => String(event?.type || '') === 'chat-abort-requested') + + addTimeline('chat-completion-poll', { + sending, + messageCount, + streamingLength: Number(snapshot.chat?.streamingText?.length || 0), + lastMessageRole, + turnCompleted: hasTurnCompletion, + toolCallCount: Number(snapshot.chat?.lastTurnComplete?.toolCallCount || 0), + toolResultCount: Number(snapshot.chat?.lastTurnComplete?.toolResultCount || 0), + abortedByUser, + }) + + if (!sending && messageCount > baselineMessageCount && hasTurnCompletion) { + return snapshot + } + + if (!sending && lastMessageRole === 'error') { + return snapshot + } + + if (!sending && abortedByUser) { + return snapshot + } + + return undefined + }, 180_000, 1_000) + + report.final = { + providerConfigured: Boolean(finalSnapshot.provider?.configured), + providerId: String(finalSnapshot.provider?.activeProvider || ''), + modelId: String(finalSnapshot.provider?.activeModel || ''), + messageCount: Number(finalSnapshot.chat?.messageCount || 0), + lastMessageRole: String(finalSnapshot.chat?.lastMessage?.role || ''), + lastMessageText: summarizeMessageText(finalSnapshot.chat?.lastMessage?.text), + lastTurnOutput: summarizeMessageText(finalSnapshot.chat?.lastTurnComplete?.outputText), + toolCallCount: Number(finalSnapshot.chat?.lastTurnComplete?.toolCallCount || 0), + toolResultCount: Number(finalSnapshot.chat?.lastTurnComplete?.toolResultCount || 0), + } + + if (report.final.lastMessageRole === 'error') { + throw new Error(`AIRI chat failed on ${report.final.providerId}/${report.final.modelId}: ${report.final.lastMessageText || 'unknown error'}`) + } + + const configuredSnapshot = await waitFor('discord UI configured', async () => { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + pushSnapshot('discord-ui-poll', snapshot) + + const tokenLength = Number(snapshot.discord?.tokenLength || 0) + const configured = Boolean(snapshot.discord?.configured) + const enabledState = Boolean(snapshot.discord?.enabled) + const targetTokenSatisfied = report.discord.expectedTokenLength > 0 + ? tokenLength === report.discord.expectedTokenLength + : tokenLength > 0 + + addTimeline('discord-ui-poll', { + route: String(snapshot.route || ''), + enabled: enabledState, + configured, + tokenLength, + }) + + return enabledState && configured && targetTokenSatisfied ? snapshot : undefined + }, 90_000, 1_000) + addTimeline('discord-ui-configured', { + enabled: Boolean(configuredSnapshot.discord?.enabled), + configured: Boolean(configuredSnapshot.discord?.configured), + tokenLength: Number(configuredSnapshot.discord?.tokenLength || 0), + route: String(configuredSnapshot.route || ''), + }) + + if (verifyDiscordBot) { + const botOutcome = await waitFor('discord bot configuration outcome', async () => { + if (discordRuntimeState.connected) { + return { + status: 'connected', + } + } + + if (allowLoginFailure && discordRuntimeState.receivedConfig && discordRuntimeState.attemptedConnect && discordRuntimeState.applyFailure) { + return { + status: 'login-failed-but-allowed', + } + } + + return undefined + }, 90_000, 500) + addTimeline('discord-bot-outcome', botOutcome) + } + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'chat-final' }, + }) + addTimeline('screenshot-captured', { label: 'chat-final' }) + + const desktopState = await mcpClient.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + report.mcp.desktopState = requireStructuredContent(desktopState, 'desktop_get_state') + + const sessionTrace = await mcpClient.callTool({ + name: 'desktop_get_session_trace', + arguments: { limit: 200 }, + }) + report.mcp.sessionTrace = requireStructuredContent(sessionTrace, 'desktop_get_session_trace') + + report.discord.ui = { + route: String(configuredSnapshot.route || ''), + enabled: Boolean(configuredSnapshot.discord?.enabled), + configured: Boolean(configuredSnapshot.discord?.configured), + tokenLength: Number(configuredSnapshot.discord?.tokenLength || 0), + } + report.discord.bot = { + ...discordRuntimeState, + } + + if (verifyDiscordBot && !allowLoginFailure && !discordRuntimeState.connected) { + throw new Error('Discord bot did not finish connecting. Provide a valid Discord bot token or rerun with AIRI_E2E_DISCORD_ALLOW_LOGIN_FAILURE=true for plumbing-only validation.') + } + + if (report.paths.auditLogPath) { + const audit = await readFile(report.paths.auditLogPath, 'utf-8').catch(() => '') + addTimeline('audit-log-summary', { + lineCount: audit ? audit.trim().split('\n').filter(Boolean).length : 0, + }) + } + + report.status = 'completed' + await writeReport() + + console.info(JSON.stringify({ + ok: true, + reportPath, + providerConfigured: report.final.providerConfigured, + providerId: report.final.providerId, + modelId: report.final.modelId, + lastMessageRole: report.final.lastMessageRole, + lastMessageText: report.final.lastMessageText, + lastTurnOutput: report.final.lastTurnOutput, + toolCallCount: report.final.toolCallCount, + toolResultCount: report.final.toolResultCount, + discordUiConfigured: report.discord.ui?.configured, + discordUiEnabled: report.discord.ui?.enabled, + tokenLength: report.discord.ui?.tokenLength, + discordBotConnected: report.discord.bot?.connected, + discordBotReadyUserTag: report.discord.bot?.readyUserTag, + discordBotApplyFailure: report.discord.bot?.applyFailure, + allowLoginFailure, + auditLogPath: report.paths.auditLogPath, + screenshotsDir: report.paths.screenshotsDir, + }, null, 2)) + } + catch (error) { + report.status = 'failed' + report.discord.bot = { + ...discordRuntimeState, + } + report.error = error instanceof Error ? error.stack || error.message : String(error) + addTimeline('failure', { error: report.error }) + await writeReport() + console.error(report.error) + exitCode = 1 + } + finally { + if (chatTargetClient && !chatClientSharesMainTarget) { + await chatTargetClient.close().catch(() => {}) + } + await mainTargetClient?.close().catch(() => {}) + await mcpClient?.close().catch(() => {}) + + if (discordBotProcess && !discordBotProcess.killed) { + discordBotProcess.kill('SIGINT') + await sleep(1_500) + if (discordBotProcess.exitCode == null) { + discordBotProcess.kill('SIGTERM') + } + } + + if (stageProcess && !stageProcess.killed) { + stageProcess.kill('SIGINT') + await sleep(1_500) + if (stageProcess.exitCode == null) { + stageProcess.kill('SIGTERM') + } + } + + await writeReport().catch(() => {}) + } +} + +main() + .catch((error) => { + const message = error instanceof Error ? error.stack || error.message : String(error) + console.error(message) + exitCode = 1 + }) + .finally(() => { + exit(exitCode) + }) diff --git a/services/computer-use-mcp/src/bin/e2e-airi-discord-observable.ts b/services/computer-use-mcp/src/bin/e2e-airi-discord-observable.ts new file mode 100644 index 000000000..38f4d1379 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-airi-discord-observable.ts @@ -0,0 +1,1057 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' + +import type { AiriDebugSnapshotLike } from '../e2e/debug-targets' + +import { execFile, spawn } from 'node:child_process' +import { createWriteStream } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +import WebSocket from 'ws' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +import { + prioritizeInspectableAiriTargets, +} from '../e2e/debug-targets' + +interface DebugTarget { + id: string + title: string + type: string + url: string + webSocketDebuggerUrl?: string +} + +interface TimelineEntry { + at: string + event: string + detail?: Record +} + +interface DiscordBotRuntimeState { + attemptedConnect: boolean + connected: boolean + receivedConfig: boolean + readyUserTag?: string + waitingForConfiguration: boolean + applyFailure?: string +} + +interface ReportShape { + startedAt: string + finishedAt?: string + status: 'running' | 'completed' | 'failed' + scenario: 'discord-enable' + reportDir: string + paths: { + reportPath: string + stageLogPath: string + discordBotLogPath: string + mcpSessionRoot: string + auditLogPath?: string + screenshotsDir?: string + } + timeline: TimelineEntry[] + debugSnapshots: unknown[] + mcp: { + capabilities?: unknown + desktopState?: unknown + sessionTrace?: unknown + } + discord: { + allowLoginFailure: boolean + expectedTokenLength: number + providerServerUrl: string + ui?: { + route?: string + enabled?: boolean + configured?: boolean + tokenLength?: number + } + bot?: DiscordBotRuntimeState + } + error?: string +} + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const repoDir = resolve(packageDir, '../..') +const preferredDebugPort = Number(env.AIRI_E2E_DEBUG_PORT || '9222') +const runId = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-') +const reportDir = resolve(packageDir, '.computer-use-mcp', 'reports', `airi-discord-observable-${runId}`) +const reportPath = resolve(reportDir, 'report.json') +const stageLogPath = resolve(reportDir, 'stage-tamagotchi.log') +const discordBotLogPath = resolve(reportDir, 'discord-bot.log') +const mcpSessionRoot = resolve(reportDir, 'computer-use-session') +const rootEnvPath = resolve(repoDir, '.env') + +const execFileAsync = promisify(execFile) + +const report: ReportShape = { + startedAt: new Date().toISOString(), + status: 'running', + scenario: 'discord-enable', + reportDir, + paths: { + reportPath, + stageLogPath, + discordBotLogPath, + mcpSessionRoot, + }, + timeline: [], + debugSnapshots: [], + mcp: {}, + discord: { + allowLoginFailure: false, + expectedTokenLength: 0, + providerServerUrl: env.AIRI_URL || 'ws://localhost:6121/ws', + }, +} + +function addTimeline(event: string, detail?: Record) { + report.timeline.push({ + at: new Date().toISOString(), + event, + detail, + }) +} + +function parseBooleanEnv(value: string | undefined, fallback = false) { + if (!value?.trim()) { + return fallback + } + + return ['1', 'true', 'yes', 'on'].includes(value.trim().toLowerCase()) +} + +function parseCommandArgs(raw: string | undefined, fallback: string[]) { + if (!raw?.trim()) { + return fallback + } + + return raw + .split(/\s+/) + .map(item => item.trim()) + .filter(Boolean) +} + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') { + throw new Error(`${label} did not return an object result`) + } + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') { + throw new Error(`${label} missing structuredContent`) + } + + return structuredContent as Record +} + +function sleep(ms: number) { + return new Promise(resolvePromise => setTimeout(resolvePromise, ms)) +} + +async function withTimeout(label: string, task: Promise, timeoutMs: number) { + let timeoutHandle: NodeJS.Timeout | undefined + + try { + return await Promise.race([ + task, + new Promise((_resolvePromise, rejectPromise) => { + timeoutHandle = setTimeout(() => rejectPromise(new Error(`Timed out waiting for ${label}`)), timeoutMs) + }), + ]) + } + finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle) + } + } +} + +async function writeReport() { + report.finishedAt = new Date().toISOString() + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf-8') +} + +async function canListenOnPort(port: number) { + return await new Promise((resolvePromise) => { + const server = createServer() + server.once('error', () => { + resolvePromise(false) + }) + server.listen(port, '127.0.0.1', () => { + server.close(() => resolvePromise(true)) + }) + }) +} + +async function findAvailablePort(preferredPort: number, attempts = 20) { + for (let index = 0; index < attempts; index += 1) { + const candidate = preferredPort + index + if (await canListenOnPort(candidate)) { + return candidate + } + } + + throw new Error(`Could not find an available remote debug port starting from ${preferredPort}`) +} + +async function terminateExistingStageTamagotchiInstances() { + const patterns = [ + resolve(repoDir, 'apps', 'stage-tamagotchi'), + '@proj-airi/stage-tamagotchi', + resolve(repoDir, 'node_modules', '.pnpm', 'electron@'), + ] + + for (const pattern of patterns) { + await execFileAsync('pkill', ['-f', pattern]).catch(() => {}) + } + + await sleep(1_500) +} + +async function terminateExistingDiscordBotInstances() { + const patterns = [ + resolve(repoDir, 'services', 'discord-bot'), + '@proj-airi/discord-bot', + ] + + for (const pattern of patterns) { + await execFileAsync('pkill', ['-f', pattern]).catch(() => {}) + } + + await sleep(1_000) +} + +async function waitFor(label: string, task: () => Promise, timeoutMs = 60_000, intervalMs = 500) { + const startedAt = Date.now() + + while ((Date.now() - startedAt) < timeoutMs) { + const value = await task() + if (value !== undefined) { + return value + } + + await sleep(intervalMs) + } + + throw new Error(`Timed out waiting for ${label}`) +} + +function parseDotEnv(text: string) { + const values: Record = {} + + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#')) { + continue + } + + const separatorIndex = trimmed.indexOf('=') + if (separatorIndex <= 0) { + continue + } + + const key = trimmed.slice(0, separatorIndex).trim() + const rawValue = trimmed.slice(separatorIndex + 1).trim() + const unwrapped = rawValue.replace(/^['"]|['"]$/gu, '') + values[key] = unwrapped + } + + return values +} + +async function readRootEnvValues() { + try { + const raw = await readFile(rootEnvPath, 'utf-8') + return parseDotEnv(raw) + } + catch { + return {} + } +} + +function resolveConfigValue(name: string, fallbackValues: Record) { + const processValue = env[name]?.trim() + if (processValue) { + return processValue + } + + const fileValue = fallbackValues[name]?.trim() + if (fileValue) { + return fileValue + } + + return '' +} + +function looksLikePlaceholderSecret(value: string) { + const normalized = value.trim().toLowerCase() + if (!normalized) { + return true + } + + return normalized.includes('replace') + || normalized.includes('placeholder') + || normalized.includes('example') + || normalized.includes('your-') + || normalized === 'changeme' +} + +function createLineListener(onLine: (line: string) => void) { + let buffer = '' + + return (chunk: { toString: (encoding: string) => string }) => { + buffer += chunk.toString('utf-8') + const lines = buffer.split(/\r?\n/u) + buffer = lines.pop() ?? '' + + for (const line of lines) { + const trimmed = line.trim() + if (trimmed) { + onLine(trimmed) + } + } + } +} + +class CdpClient { + private ws: any + private nextId = 0 + private pending = new Map void, reject: (error: Error) => void }>() + + static async connectToUrl(webSocketUrl: string, options: { enableRuntime?: boolean, enablePage?: boolean } = {}) { + const client = new CdpClient() + client.ws = new WebSocket(webSocketUrl) + + await new Promise((resolvePromise, rejectPromise) => { + const onOpen = () => resolvePromise() + const onError = (error: Error) => rejectPromise(error) + + client.ws.addEventListener('open', onOpen, { once: true }) + client.ws.addEventListener('error', onError, { once: true }) + }) + + client.ws.addEventListener('message', (event: { data: string }) => { + const payload = JSON.parse(event.data) + if (typeof payload.id === 'number') { + const pending = client.pending.get(payload.id) + if (!pending) { + return + } + + client.pending.delete(payload.id) + if (payload.error) { + pending.reject(new Error(String(payload.error.message || 'Unknown CDP error'))) + return + } + + pending.resolve(payload.result) + } + }) + + if (options.enableRuntime !== false) { + await client.send('Runtime.enable') + } + + if (options.enablePage !== false) { + await client.send('Page.enable') + } + + return client + } + + static async connect(target: DebugTarget) { + if (!target.webSocketDebuggerUrl) { + throw new Error(`Debug target ${target.title || target.id} does not expose webSocketDebuggerUrl`) + } + + return await CdpClient.connectToUrl(target.webSocketDebuggerUrl) + } + + async send(method: string, params?: Record) { + const id = ++this.nextId + const payload = { id, method, params } + + return await new Promise((resolvePromise, rejectPromise) => { + this.pending.set(id, { resolve: resolvePromise, reject: rejectPromise }) + this.ws.send(JSON.stringify(payload)) + }) + } + + async evaluate(expression: string): Promise { + const result = await this.send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + userGesture: true, + }) + + if (result?.exceptionDetails) { + const text = result.exceptionDetails.text || 'Runtime.evaluate exception' + throw new Error(String(text)) + } + + return result?.result?.value as T + } + + async close() { + if (this.ws?.readyState === 1) { + this.ws.close() + } + } +} + +async function listDebugTargets(browserWsUrl: string) { + const browserClient = await CdpClient.connectToUrl(browserWsUrl, { + enableRuntime: false, + enablePage: false, + }) + + try { + const result = await browserClient.send('Target.getTargets') as { targetInfos?: Array> } + const targetInfos = Array.isArray(result.targetInfos) ? result.targetInfos : [] + + return targetInfos + .filter(target => target.type === 'page') + .map((target) => { + const targetId = String(target.targetId || '') + return { + id: targetId, + title: String(target.title || ''), + type: String(target.type || ''), + url: String(target.url || ''), + webSocketDebuggerUrl: browserWsUrl.replace(/\/devtools\/browser\/[^/]+$/, `/devtools/page/${targetId}`), + } satisfies DebugTarget + }) + } + finally { + await browserClient.close().catch(() => {}) + } +} + +async function bringTargetToFront(client: CdpClient, label: string) { + await client.send('Page.bringToFront') + addTimeline('target-brought-to-front', { label }) + await sleep(750) +} + +async function getAiriDebugSnapshot(client: CdpClient) { + return await client.evaluate(`(() => { + const bridge = window.__AIRI_DEBUG__ + if (!bridge || typeof bridge.getSnapshot !== 'function') { + return undefined + } + + return bridge.getSnapshot() + })()`) +} + +async function findTargetWithAiriDebugBridge( + browserWsUrl: string, + label: string, + predicate?: (target: DebugTarget, snapshot: AiriDebugSnapshotLike) => boolean, +) { + return await waitFor(label, async () => { + const targets = prioritizeInspectableAiriTargets(await listDebugTargets(browserWsUrl).catch(() => [])) + + for (const target of targets) { + let client: CdpClient | undefined + + try { + client = await withTimeout( + `${label} connect ${target.title || target.url || target.id}`, + CdpClient.connect(target), + 2_500, + ) + const snapshot = await withTimeout( + `${label} snapshot ${target.title || target.url || target.id}`, + getAiriDebugSnapshot(client), + 2_500, + ) + if (!snapshot) { + continue + } + + if (predicate && !predicate(target, snapshot)) { + continue + } + + return { + target, + snapshot, + } + } + catch { + continue + } + finally { + await client?.close().catch(() => {}) + } + } + + return undefined + }, 90_000, 750) +} + +let exitCode = 0 + +async function main() { + let stageProcess: ChildProcessWithoutNullStreams | undefined + let discordBotProcess: ChildProcessWithoutNullStreams | undefined + let mcpClient: Client | undefined + let mainTargetClient: CdpClient | undefined + let browserWsUrl: string | undefined + const debugPort = await findAvailablePort(preferredDebugPort) + const rootEnvValues = await readRootEnvValues() + const allowLoginFailure = parseBooleanEnv(resolveConfigValue('AIRI_E2E_DISCORD_ALLOW_LOGIN_FAILURE', rootEnvValues), false) + const openDiscordClient = parseBooleanEnv(resolveConfigValue('AIRI_E2E_DISCORD_OPEN_CLIENT', rootEnvValues), true) + const discordToken = resolveConfigValue('AIRI_E2E_DISCORD_TOKEN', rootEnvValues) + || resolveConfigValue('DISCORD_TOKEN', rootEnvValues) + const discordRuntimeState: DiscordBotRuntimeState = { + attemptedConnect: false, + connected: false, + receivedConfig: false, + waitingForConfiguration: false, + } + + report.discord.allowLoginFailure = allowLoginFailure + report.discord.expectedTokenLength = discordToken.length + + if (looksLikePlaceholderSecret(discordToken)) { + throw new Error(`Discord demo requires AIRI_E2E_DISCORD_TOKEN (or DISCORD_TOKEN) in process env or ${rootEnvPath}. The current value is missing or still a placeholder.`) + } + + try { + await mkdir(reportDir, { recursive: true }) + await mkdir(mcpSessionRoot, { recursive: true }) + + addTimeline('bootstrap', { reportDir, debugPort, allowLoginFailure }) + await terminateExistingStageTamagotchiInstances() + await terminateExistingDiscordBotInstances() + addTimeline('terminated-stale-processes', { stage: true, discordBot: true }) + + const stageLogStream = createWriteStream(stageLogPath, { flags: 'a' }) + const discordBotLogStream = createWriteStream(discordBotLogPath, { flags: 'a' }) + + addTimeline('start-stage-tamagotchi') + stageProcess = spawn('pnpm', ['-F', '@proj-airi/stage-tamagotchi', 'dev'], { + cwd: repoDir, + env: { + ...env, + APP_REMOTE_DEBUG: 'true', + APP_REMOTE_DEBUG_PORT: String(debugPort), + APP_REMOTE_DEBUG_NO_OPEN: 'true', + }, + stdio: 'pipe', + }) + + const onStageChunk = createLineListener((line) => { + const match = line.match(/DevTools listening on (ws:\/\/\S+)/) + if (match?.[1]) { + browserWsUrl = match[1] + } + }) + + stageProcess.stdout.on('data', (chunk) => { + stageLogStream.write(chunk) + onStageChunk(chunk) + }) + stageProcess.stderr.on('data', (chunk) => { + stageLogStream.write(chunk) + onStageChunk(chunk) + }) + + stageProcess.on('exit', (code, signal) => { + addTimeline('stage-tamagotchi-exit', { + code: code ?? undefined, + signal: signal ?? undefined, + }) + }) + + const activeBrowserWsUrl = await waitFor('remote debug browser websocket', async () => { + return browserWsUrl + }, 120_000, 500) + addTimeline('remote-debug-browser-ready', { browserWsUrl: activeBrowserWsUrl, debugPort }) + + const mainTargetMatch = await findTargetWithAiriDebugBridge( + activeBrowserWsUrl, + 'AIRI main target', + ) + const mainTarget = mainTargetMatch.target + addTimeline('main-target-ready', { + title: mainTarget.title, + url: mainTarget.url, + route: mainTargetMatch.snapshot.route, + documentTitle: mainTargetMatch.snapshot.documentTitle, + }) + + mainTargetClient = await CdpClient.connect(mainTarget) + await bringTargetToFront(mainTargetClient, 'main') + + addTimeline('start-discord-bot') + discordBotProcess = spawn('pnpm', ['-F', '@proj-airi/discord-bot', 'start'], { + cwd: repoDir, + env: { + ...env, + DISCORD_TOKEN: '', + AIRI_TOKEN: env.AIRI_TOKEN || 'abcd', + AIRI_URL: report.discord.providerServerUrl, + }, + stdio: 'pipe', + }) + + const onDiscordBotChunk = createLineListener((line) => { + if (line.includes('Waiting for configuration from UI')) { + discordRuntimeState.waitingForConfiguration = true + addTimeline('discord-bot-waiting-for-ui-config') + } + if (line.includes('Received Discord configuration:')) { + discordRuntimeState.receivedConfig = true + addTimeline('discord-bot-received-config') + } + if (line.includes('Connecting Discord client...')) { + discordRuntimeState.attemptedConnect = true + addTimeline('discord-bot-connecting') + } + if (line.includes('Discord client connected.')) { + discordRuntimeState.connected = true + addTimeline('discord-bot-connected') + } + if (line.includes('Discord bot ready! User:')) { + discordRuntimeState.connected = true + discordRuntimeState.readyUserTag = line.split('Discord bot ready! User:').at(1)?.trim() || undefined + addTimeline('discord-bot-ready', { + userTag: discordRuntimeState.readyUserTag, + }) + } + if (line.includes('Failed to apply Discord configuration.')) { + discordRuntimeState.applyFailure = line + addTimeline('discord-bot-apply-failure', { line }) + } + }) + + discordBotProcess.stdout.on('data', (chunk) => { + discordBotLogStream.write(chunk) + onDiscordBotChunk(chunk) + }) + discordBotProcess.stderr.on('data', (chunk) => { + discordBotLogStream.write(chunk) + onDiscordBotChunk(chunk) + }) + + discordBotProcess.on('exit', (code, signal) => { + addTimeline('discord-bot-exit', { + code: code ?? undefined, + signal: signal ?? undefined, + }) + }) + + await waitFor('discord bot startup', async () => { + return discordRuntimeState.waitingForConfiguration ? true : undefined + }, 45_000, 500) + + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start']) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'macos-local', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_OPENABLE_APPS: 'Terminal,Cursor,Google Chrome,Electron,Discord', + COMPUTER_USE_DENY_APPS: '1Password,Keychain,System Settings,Activity Monitor', + COMPUTER_USE_SESSION_TAG: `airi-discord-e2e-${runId}`, + COMPUTER_USE_ALLOWED_BOUNDS: env.COMPUTER_USE_ALLOWED_BOUNDS || '0,0,2560,1600', + COMPUTER_USE_SESSION_ROOT: mcpSessionRoot, + }, + stderr: 'pipe', + }) + + mcpClient = new Client({ + name: '@proj-airi/computer-use-mcp-e2e-airi-discord', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk: { toString: (encoding: string) => string }) => { + const text = chunk.toString('utf-8').trim() + if (text) { + addTimeline('computer-use-mcp-stderr', { text }) + } + }) + + await mcpClient.connect(transport) + addTimeline('computer-use-mcp-connected') + + const capabilities = await mcpClient.callTool({ + name: 'desktop_get_capabilities', + arguments: {}, + }) + const capabilitiesData = requireStructuredContent(capabilities, 'desktop_get_capabilities') + report.mcp.capabilities = capabilitiesData + report.paths.auditLogPath = String((capabilitiesData.session as Record | undefined)?.auditLogPath || '') || undefined + report.paths.screenshotsDir = String((capabilitiesData.session as Record | undefined)?.screenshotsDir || '') || undefined + addTimeline('desktop-capabilities', { + executionMode: (capabilitiesData.executionTarget as Record | undefined)?.mode, + auditLogPath: report.paths.auditLogPath, + screenshotsDir: report.paths.screenshotsDir, + }) + + await mcpClient.callTool({ + name: 'desktop_focus_app', + arguments: { app: 'Electron' }, + }) + addTimeline('desktop-focus-app', { app: 'Electron' }) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'discord-before-route' }, + }) + addTimeline('screenshot-captured', { label: 'discord-before-route' }) + + await mainTargetClient.evaluate(`window.__AIRI_DEBUG__.navigateTo('/settings/modules/messaging-discord')`) + addTimeline('navigate-to-discord-settings') + + const settingsSnapshot = await waitFor('discord settings route', async () => { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + report.debugSnapshots.push(snapshot) + + const onRoute = String(snapshot.route || '').includes('/settings/modules/messaging-discord') + const hasControls = Boolean(snapshot.discord?.hasTokenInput) + && Boolean(snapshot.discord?.hasSaveButton) + + return onRoute && hasControls ? snapshot : undefined + }, 30_000, 500) + addTimeline('discord-settings-ready', { + route: String(settingsSnapshot.route || ''), + enabled: Boolean(settingsSnapshot.discord?.enabled), + configured: Boolean(settingsSnapshot.discord?.configured), + }) + + await mcpClient.callTool({ + name: 'desktop_observe_windows', + arguments: { limit: 24 }, + }) + addTimeline('desktop-observed-windows') + + if (!settingsSnapshot.discord?.enabled) { + const checkboxFocused = await waitFor('discord checkbox focus', async () => { + const state = await mainTargetClient!.evaluate>(`(() => { + const checkbox = document.querySelector('input[type="checkbox"], [role="switch"], button[aria-checked], button[data-state]') + if (!(checkbox instanceof HTMLElement)) { + return { ok: false } + } + + checkbox.focus() + return { + ok: document.activeElement === checkbox, + checked: checkbox instanceof HTMLInputElement ? checkbox.checked : checkbox.getAttribute('aria-checked') === 'true', + role: checkbox.getAttribute('role') || '', + tagName: checkbox.tagName, + } + })()`) + + return state.ok ? state : undefined + }, 10_000, 250) + addTimeline('discord-checkbox-focused', { + checked: Boolean(checkboxFocused.checked), + role: String(checkboxFocused.role || ''), + tagName: String(checkboxFocused.tagName || ''), + }) + + const toggle = await mcpClient.callTool({ + name: 'desktop_press_keys', + arguments: { + keys: ['space'], + captureAfter: true, + }, + }) + const toggleData = requireStructuredContent(toggle, 'desktop_press_keys') + addTimeline('discord-checkbox-toggled', { + status: toggleData.status, + screenshotPath: (toggleData.screenshot as Record | undefined)?.path, + }) + + const enabledSnapshot = await waitFor('discord enabled state', async () => { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + report.debugSnapshots.push(snapshot) + return snapshot.discord?.enabled ? snapshot : undefined + }, 5_000, 250).catch(async () => { + addTimeline('discord-toggle-keyboard-fallback') + + const fallbackSnapshot = await mainTargetClient!.evaluate>(`(() => { + const checkbox = document.querySelector('input[type="checkbox"], [role="switch"], button[aria-checked], button[data-state]') + if (!(checkbox instanceof HTMLElement)) { + throw new Error('Discord toggle not found') + } + + checkbox.click() + return window.__AIRI_DEBUG__.getSnapshot() + })()`) + + report.debugSnapshots.push(fallbackSnapshot) + + return await waitFor('discord enabled state after fallback click', async () => { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + report.debugSnapshots.push(snapshot) + return snapshot.discord?.enabled ? snapshot : undefined + }, 5_000, 250) + }) + + addTimeline('discord-enabled-confirmed', { + enabled: Boolean(enabledSnapshot.discord?.enabled), + }) + } + else { + addTimeline('discord-already-enabled') + } + + const tokenAppliedSnapshot = await mainTargetClient.evaluate>(`(() => { + const input = document.querySelector('input[type="password"]') + if (!(input instanceof HTMLInputElement)) { + throw new Error('Discord token input not found') + } + + input.focus() + input.value = ${JSON.stringify(discordToken)} + input.dispatchEvent(new Event('input', { bubbles: true })) + input.dispatchEvent(new Event('change', { bubbles: true })) + + return window.__AIRI_DEBUG__.getSnapshot() + })()`) + report.debugSnapshots.push(tokenAppliedSnapshot) + addTimeline('discord-token-applied', { + tokenLength: discordToken.length, + appliedVia: 'renderer-evaluate', + }) + + const tokenSnapshot = await waitFor('discord token to settle', async () => { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + report.debugSnapshots.push(snapshot) + return Number(snapshot.discord?.tokenLength || 0) === discordToken.length ? snapshot : undefined + }, 10_000, 250) + addTimeline('discord-token-confirmed', { + tokenLength: Number(tokenSnapshot.discord?.tokenLength || 0), + }) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'discord-before-save' }, + }) + addTimeline('screenshot-captured', { label: 'discord-before-save' }) + + const saveButtonFocused = await waitFor('discord save button focus', async () => { + const state = await mainTargetClient!.evaluate>(`(() => { + const passwordInput = document.querySelector('input[type="password"]') + const buttons = Array.from(document.querySelectorAll('button')) + const button = buttons.find((candidate) => { + const text = candidate.textContent?.trim().toLowerCase() || '' + if (text === 'save' || text.includes('保存')) { + return true + } + + if (!passwordInput) { + return false + } + + return Boolean(passwordInput.compareDocumentPosition(candidate) & Node.DOCUMENT_POSITION_FOLLOWING) + }) + + if (!(button instanceof HTMLButtonElement)) { + return { ok: false } + } + + button.focus() + return { + ok: document.activeElement === button, + text: button.textContent?.trim() || '', + } + })()`) + + return state.ok ? state : undefined + }, 10_000, 250) + addTimeline('discord-save-button-focused', { + text: String(saveButtonFocused.text || ''), + }) + + const saveResult = await mcpClient.callTool({ + name: 'desktop_press_keys', + arguments: { + keys: ['enter'], + captureAfter: true, + }, + }) + const saveData = requireStructuredContent(saveResult, 'desktop_press_keys') + addTimeline('discord-save-submitted', { + status: saveData.status, + screenshotPath: (saveData.screenshot as Record | undefined)?.path, + }) + + const configuredSnapshot = await waitFor('discord configured state', async () => { + const snapshot = await mainTargetClient!.evaluate>('window.__AIRI_DEBUG__.getSnapshot()') + report.debugSnapshots.push(snapshot) + + const tokenLength = Number(snapshot.discord?.tokenLength || 0) + const configured = Boolean(snapshot.discord?.configured) + const enabledState = Boolean(snapshot.discord?.enabled) + + return enabledState && configured && tokenLength === discordToken.length ? snapshot : undefined + }, 10_000, 250) + addTimeline('discord-ui-configured', { + enabled: Boolean(configuredSnapshot.discord?.enabled), + configured: Boolean(configuredSnapshot.discord?.configured), + tokenLength: Number(configuredSnapshot.discord?.tokenLength || 0), + }) + + const botOutcome = await waitFor('discord bot configuration outcome', async () => { + if (discordRuntimeState.connected) { + return { + status: 'connected', + } + } + + if (allowLoginFailure && discordRuntimeState.receivedConfig && discordRuntimeState.attemptedConnect && discordRuntimeState.applyFailure) { + return { + status: 'login-failed-but-allowed', + } + } + + return undefined + }, 60_000, 500) + addTimeline('discord-bot-outcome', botOutcome) + + if (openDiscordClient) { + try { + const openDiscordAppResult = await mcpClient.callTool({ + name: 'desktop_open_app', + arguments: { app: 'Discord' }, + }) + const openDiscordAppData = requireStructuredContent(openDiscordAppResult, 'desktop_open_app') + addTimeline('discord-client-opened', { + status: openDiscordAppData.status, + appName: openDiscordAppData.appName, + windowTitle: openDiscordAppData.windowTitle, + }) + + await mcpClient.callTool({ + name: 'desktop_observe_windows', + arguments: { limit: 24, app: 'Discord' }, + }).catch(() => undefined) + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'discord-client-opened' }, + }).catch(() => undefined) + } + catch (error) { + addTimeline('discord-client-open-skipped', { + error: error instanceof Error ? error.message : String(error), + }) + } + } + + await mcpClient.callTool({ + name: 'desktop_screenshot', + arguments: { label: 'discord-final' }, + }) + addTimeline('screenshot-captured', { label: 'discord-final' }) + + const desktopState = await mcpClient.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + report.mcp.desktopState = requireStructuredContent(desktopState, 'desktop_get_state') + + const sessionTrace = await mcpClient.callTool({ + name: 'desktop_get_session_trace', + arguments: { limit: 200 }, + }) + report.mcp.sessionTrace = requireStructuredContent(sessionTrace, 'desktop_get_session_trace') + + report.discord.ui = { + route: String(configuredSnapshot.route || ''), + enabled: Boolean(configuredSnapshot.discord?.enabled), + configured: Boolean(configuredSnapshot.discord?.configured), + tokenLength: Number(configuredSnapshot.discord?.tokenLength || 0), + } + report.discord.bot = { + ...discordRuntimeState, + } + + if (!allowLoginFailure && !discordRuntimeState.connected) { + throw new Error('Discord bot did not finish connecting. Provide a valid Discord bot token or rerun with AIRI_E2E_DISCORD_ALLOW_LOGIN_FAILURE=true for plumbing-only validation.') + } + + if (report.paths.auditLogPath) { + const audit = await readFile(report.paths.auditLogPath, 'utf-8').catch(() => '') + addTimeline('audit-log-summary', { + lineCount: audit ? audit.trim().split('\n').filter(Boolean).length : 0, + }) + } + + report.status = 'completed' + await writeReport() + + console.info(JSON.stringify({ + ok: true, + reportPath, + discordUiConfigured: report.discord.ui?.configured, + discordUiEnabled: report.discord.ui?.enabled, + tokenLength: report.discord.ui?.tokenLength, + discordBotConnected: report.discord.bot?.connected, + discordBotReadyUserTag: report.discord.bot?.readyUserTag, + discordBotApplyFailure: report.discord.bot?.applyFailure, + allowLoginFailure, + auditLogPath: report.paths.auditLogPath, + screenshotsDir: report.paths.screenshotsDir, + }, null, 2)) + } + catch (error) { + report.status = 'failed' + report.discord.bot = { + ...discordRuntimeState, + } + report.error = error instanceof Error ? error.stack || error.message : String(error) + addTimeline('failure', { error: report.error }) + await writeReport() + console.error(report.error) + exitCode = 1 + } + finally { + await mainTargetClient?.close().catch(() => {}) + await mcpClient?.close().catch(() => {}) + + if (discordBotProcess && !discordBotProcess.killed) { + discordBotProcess.kill('SIGINT') + await sleep(1_500) + if (discordBotProcess.exitCode == null) { + discordBotProcess.kill('SIGTERM') + } + } + + if (stageProcess && !stageProcess.killed) { + stageProcess.kill('SIGINT') + await sleep(1_500) + if (stageProcess.exitCode == null) { + stageProcess.kill('SIGTERM') + } + } + + await writeReport().catch(() => {}) + } +} + +main().finally(() => { + exit(exitCode) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-browser-reroute.ts b/services/computer-use-mcp/src/bin/e2e-browser-reroute.ts new file mode 100644 index 000000000..232f6abc9 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-browser-reroute.ts @@ -0,0 +1,231 @@ +/** + * Secondary regression script: workflow reroute path. + * + * This script intentionally exercises a deterministic reroute-producing + * workflow path and then follows the suggested tool. It is stricter than + * a smoke test, but it is not yet used to claim browser dual-stack + * product support because the current dry-run fixture reroutes through a + * stable accessibility path rather than a guaranteed browser surface. + * + * Usage: + * pnpm -F @proj-airi/computer-use-mcp exec tsx ./src/bin/e2e-browser-reroute.ts + */ + +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assert(condition: boolean, message: string): asserts condition { + if (!condition) + throw new Error(`Assertion failed: ${message}`) +} + +function requireStructuredContent(result: unknown, label: string): Record { + if (!result || typeof result !== 'object') + throw new Error(`${label}: result is not an object`) + + const sc = (result as { structuredContent?: unknown }).structuredContent + if (!sc || typeof sc !== 'object') + throw new Error(`${label}: missing structuredContent`) + + return sc as Record +} + +async function createClient(): Promise { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = (env.COMPUTER_USE_SMOKE_SERVER_ARGS || 'start').split(/\s+/).filter(Boolean) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'dry-run', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_SESSION_TAG: 'e2e-browser-reroute', + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,1920,1080', + COMPUTER_USE_OPENABLE_APPS: 'Google Chrome,Firefox,Safari', + }, + stderr: 'pipe', + }) + + const client = new Client({ + name: '@proj-airi/e2e-browser-reroute', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[stderr] ${text}`) + }) + + await client.connect(transport) + return client +} + +// --------------------------------------------------------------------------- +// Test phases +// --------------------------------------------------------------------------- + +async function phase1_triggerReroute(client: Client): Promise> { + console.info('\n── Phase 1: Trigger reroute via workflow_browse_and_act ──') + + const result = await client.callTool({ + name: 'workflow_browse_and_act', + arguments: { + app: 'Google Chrome', + goal: 'Check the homepage', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_browse_and_act') + console.info(` Status: ${data.status}`) + assert(data.kind === 'workflow_reroute', `expected workflow_reroute kind, got ${String(data.kind)}`) + assert(data.status === 'reroute_required', `expected reroute_required, got ${String(data.status)}`) + console.info(' → Reroute detected') + return data +} + +function phase2_verifyRerouteContract(data: Record) { + console.info('\n── Phase 2: Verify reroute contract shape ──') + + if (data.kind !== 'workflow_reroute') { + console.info(' (Skipped — no reroute to verify)') + return null + } + + assert(data.status === 'reroute_required', `expected reroute_required, got ${data.status}`) + assert(typeof data.workflow === 'string', 'reroute.workflow must be a string') + + const reroute = data.reroute + assert(reroute != null && typeof reroute === 'object', 'reroute detail must be an object') + + const r = reroute as Record + assert(typeof r.recommendedSurface === 'string', 'recommendedSurface must be a string') + assert(typeof r.suggestedTool === 'string', 'suggestedTool must be a string') + assert(typeof r.strategyReason === 'string', 'strategyReason must be a string') + assert(typeof r.explanation === 'string', 'explanation must be a string') + + // Optional fields: only verify type if present + if (r.executionReason !== undefined) { + assert(typeof r.executionReason === 'string', 'executionReason must be a string when present') + } + if (r.availableSurfaces !== undefined) { + assert(Array.isArray(r.availableSurfaces), 'availableSurfaces must be an array when present') + } + if (r.preferredSurface !== undefined) { + assert(typeof r.preferredSurface === 'string', 'preferredSurface must be a string when present') + } + + console.info(` ✓ kind: ${data.kind}`) + console.info(` ✓ workflow: ${data.workflow}`) + console.info(` ✓ recommendedSurface: ${r.recommendedSurface}`) + console.info(` ✓ suggestedTool: ${r.suggestedTool}`) + console.info(` ✓ strategyReason: ${r.strategyReason}`) + + return r +} + +async function phase3_followReroute(client: Client, reroute: Record | null) { + console.info('\n── Phase 3: Follow reroute to suggested surface ──') + + assert(reroute != null, 'reroute detail must be present before following suggested tool') + + const suggestedTool = String(reroute.suggestedTool) + console.info(` Following reroute → calling ${suggestedTool}`) + + // Verify the suggested tool actually exists + const { tools } = await client.listTools() + const toolNames = new Set(tools.map(t => t.name)) + assert(toolNames.has(suggestedTool), `suggested tool ${suggestedTool} not registered`) + console.info(` ✓ ${suggestedTool} is registered`) + + const result = await client.callTool({ + name: suggestedTool, + arguments: {}, + }) + + assert(result && typeof result === 'object', `${suggestedTool} returned an invalid result`) + const sc = (result as { structuredContent?: unknown }).structuredContent + const content = (result as { content?: unknown }).content + + if (sc && typeof sc === 'object') { + const scData = sc as Record + assert(scData.status !== 'error', `${suggestedTool} returned structuredContent.status=error`) + console.info(` ✓ ${suggestedTool} returned structuredContent (status: ${scData.status ?? 'n/a'})`) + return + } + + assert(Array.isArray(content), `${suggestedTool} must return structuredContent or a content array`) + assert(content.length > 0, `${suggestedTool} content array must not be empty`) + console.info(` ✓ ${suggestedTool} returned content array (${content.length} parts)`) +} + +async function phase4_desktopState(client: Client) { + console.info('\n── Phase 4: Verify desktop state after reroute flow ──') + + const result = await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'desktop_get_state') + assert(data.status === 'ok', `get_state: expected ok, got ${data.status}`) + + const runState = data.runState as Record + console.info(` Active app: ${runState.activeApp ?? 'none'}`) + console.info(` ✓ State is consistent after reroute flow`) +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.info('╔═══════════════════════════════════════════════════════╗') + console.info('║ Secondary Regression: Workflow Reroute Path ║') + console.info('╚═══════════════════════════════════════════════════════╝') + + const client = await createClient() + + try { + // Verify required tools + const { tools } = await client.listTools() + const names = new Set(tools.map(t => t.name)) + for (const t of ['workflow_browse_and_act', 'desktop_get_state']) { + assert(names.has(t), `missing required tool: ${t}`) + } + console.info(` ${tools.length} tools available`) + + const browseResult = await phase1_triggerReroute(client) + const rerouteDetail = phase2_verifyRerouteContract(browseResult) + await phase3_followReroute(client, rerouteDetail) + await phase4_desktopState(client) + + console.info('\n╔═══════════════════════════════════════════════════════╗') + console.info('║ WORKFLOW REROUTE REGRESSION — PASSED ║') + console.info('╚═══════════════════════════════════════════════════════╝') + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error('\n❌ WORKFLOW REROUTE REGRESSION FAILED') + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-developer-workflow.ts b/services/computer-use-mcp/src/bin/e2e-developer-workflow.ts new file mode 100644 index 000000000..b70b6ac57 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-developer-workflow.ts @@ -0,0 +1,263 @@ +/** + * E2E release gate: Developer workflow happy path. + * + * Simulates the multi-tool chain a real chat session would produce: + * workflow_open_workspace → workflow_validate_workspace → workflow_run_tests + * + * Each step must complete successfully with a valid structuredContent shape, + * and the chain must propagate the same projectPath end-to-end. + * + * Usage: + * pnpm -F @proj-airi/computer-use-mcp exec tsx ./src/bin/e2e-developer-workflow.ts + */ + +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assert(condition: boolean, message: string): asserts condition { + if (!condition) + throw new Error(`Assertion failed: ${message}`) +} + +function requireStructuredContent(result: unknown, label: string): Record { + if (!result || typeof result !== 'object') + throw new Error(`${label}: result is not an object`) + + const sc = (result as { structuredContent?: unknown }).structuredContent + if (!sc || typeof sc !== 'object') + throw new Error(`${label}: missing structuredContent`) + + return sc as Record +} + +function createProjectDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'e2e-dev-workflow-')) + writeFileSync(join(dir, 'README.md'), '# e2e test project\n', 'utf8') + writeFileSync(join(dir, 'index.ts'), 'export const ok = true\n', 'utf8') + return dir +} + +async function createClient(): Promise { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = (env.COMPUTER_USE_SMOKE_SERVER_ARGS || 'start').split(/\s+/).filter(Boolean) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'dry-run', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_SESSION_TAG: 'e2e-developer-workflow', + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,1920,1080', + COMPUTER_USE_OPENABLE_APPS: 'Finder,Terminal,Visual Studio Code', + }, + stderr: 'pipe', + }) + + const client = new Client({ + name: '@proj-airi/e2e-developer-workflow', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[stderr] ${text}`) + }) + + await client.connect(transport) + return client +} + +function assertCompletedWorkflow(data: Record, label: string) { + assert( + data.status === 'completed', + `${label}: expected completed, got ${String(data.status)}`, + ) +} + +function requireSucceededStep( + data: Record, + workflowLabel: string, + stepLabel: string, +) { + const steps = data.stepResults as Array<{ label: string, succeeded: boolean }> + const step = steps.find(candidate => candidate.label === stepLabel) + assert(step !== undefined, `${workflowLabel}: missing expected step "${stepLabel}"`) + assert(step.succeeded === true, `${workflowLabel}: expected step "${stepLabel}" to succeed`) +} + +// --------------------------------------------------------------------------- +// Chain steps +// --------------------------------------------------------------------------- + +async function step1_openWorkspace(client: Client, projectPath: string) { + console.info('\n── Step 1: workflow_open_workspace ──') + + const result = await client.callTool({ + name: 'workflow_open_workspace', + arguments: { + projectPath, + ideApp: 'Visual Studio Code', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_open_workspace') + console.info(` Status: ${data.status}`) + assertCompletedWorkflow(data, 'workflow_open_workspace') + + const steps = data.stepResults as Array<{ label: string, succeeded: boolean }> + for (const s of steps) { + console.info(` ${s.succeeded ? '✓' : '✗'} ${s.label}`) + } + + requireSucceededStep(data, 'workflow_open_workspace', 'Reveal project in Finder') + requireSucceededStep(data, 'workflow_open_workspace', 'Open project in Visual Studio Code') + + return data +} + +async function step2_validateWorkspace(client: Client, projectPath: string) { + console.info('\n── Step 2: workflow_validate_workspace ──') + + const result = await client.callTool({ + name: 'workflow_validate_workspace', + arguments: { + projectPath, + ideApp: 'Visual Studio Code', + changesCommand: 'printf " M index.ts\\n"', + checkCommand: 'echo "typecheck ok"', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_validate_workspace') + console.info(` Status: ${data.status}`) + assertCompletedWorkflow(data, 'workflow_validate_workspace') + + const steps = data.stepResults as Array<{ label: string, succeeded: boolean }> + for (const s of steps) { + console.info(` ${s.succeeded ? '✓' : '✗'} ${s.label}`) + } + + requireSucceededStep(data, 'workflow_validate_workspace', 'Confirm project working directory') + requireSucceededStep(data, 'workflow_validate_workspace', 'Inspect local changes') + requireSucceededStep(data, 'workflow_validate_workspace', 'Run workspace validation') + + return data +} + +async function step3_runTests(client: Client, projectPath: string) { + console.info('\n── Step 3: workflow_run_tests ──') + + const result = await client.callTool({ + name: 'workflow_run_tests', + arguments: { + projectPath, + testCommand: 'echo "all tests passed"', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_run_tests') + console.info(` Status: ${data.status}`) + assertCompletedWorkflow(data, 'workflow_run_tests') + + const steps = data.stepResults as Array<{ label: string, succeeded: boolean }> + for (const s of steps) { + console.info(` ${s.succeeded ? '✓' : '✗'} ${s.label}`) + } + + requireSucceededStep(data, 'workflow_run_tests', 'Change directory to project root') + requireSucceededStep(data, 'workflow_run_tests', 'Run test suite') + + return data +} + +async function step4_verifyStateReflectsChain(client: Client, projectPath: string) { + console.info('\n── Step 4: desktop_get_state (chain summary) ──') + + const result = await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'desktop_get_state') + assert(data.status === 'ok', `get_state: expected ok, got ${data.status}`) + + const runState = data.runState as Record + const terminalState = runState.terminalState as Record | undefined + + console.info(` Active app: ${runState.activeApp ?? 'none'}`) + console.info(` Terminal state: ${JSON.stringify(terminalState)}`) + + assert(terminalState != null, 'desktop_get_state: terminalState must be present after developer chain') + assert(terminalState.effectiveCwd === projectPath, `desktop_get_state: expected effectiveCwd=${projectPath}, got ${String(terminalState.effectiveCwd)}`) + assert(terminalState.lastExitCode === 0, `desktop_get_state: expected lastExitCode=0, got ${String(terminalState.lastExitCode)}`) + assert( + typeof terminalState.lastCommandSummary === 'string' && terminalState.lastCommandSummary.includes('all tests passed'), + `desktop_get_state: expected lastCommandSummary to include "all tests passed", got ${String(terminalState.lastCommandSummary)}`, + ) + + return data +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.info('╔═══════════════════════════════════════════════════════╗') + console.info('║ E2E Release Gate: Developer Workflow Happy Path ║') + console.info('╚═══════════════════════════════════════════════════════╝') + + const projectPath = createProjectDir() + console.info(` Project directory: ${projectPath}`) + + const client = await createClient() + + try { + // Verify tools are present + const { tools } = await client.listTools() + const names = new Set(tools.map(t => t.name)) + for (const t of ['workflow_open_workspace', 'workflow_validate_workspace', 'workflow_run_tests', 'desktop_get_state']) { + assert(names.has(t), `missing required tool: ${t}`) + } + console.info(` ${tools.length} tools available`) + + // Run the chain + await step1_openWorkspace(client, projectPath) + await step2_validateWorkspace(client, projectPath) + await step3_runTests(client, projectPath) + await step4_verifyStateReflectsChain(client, projectPath) + + console.info('\n╔═══════════════════════════════════════════════════════╗') + console.info('║ DEVELOPER WORKFLOW E2E — ALL STEPS PASSED ║') + console.info('╚═══════════════════════════════════════════════════════╝') + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error('\n❌ E2E DEVELOPER WORKFLOW FAILED') + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-terminal-exec.ts b/services/computer-use-mcp/src/bin/e2e-terminal-exec.ts new file mode 100644 index 000000000..4a6f8b5e1 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-terminal-exec.ts @@ -0,0 +1,260 @@ +/** + * Real E2E: Terminal exec happy path. + * + * Proves the terminal_exec surface works end-to-end through a real + * MCP stdio transport: + * + * 1. Open workspace (dry-run desktop, real terminal) + * 2. Run real shell commands (pwd, echo) + * 3. Verify terminal state is written back after each step + * 4. Agent can continue based on results + * + * Unlike the mocked integration tests, this exercises the real + * `createLocalShellRunner` backed by `child_process.spawn`. + * + * Usage: + * pnpm -F @proj-airi/computer-use-mcp e2e:terminal-exec + */ + +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assert(condition: boolean, message: string): asserts condition { + if (!condition) + throw new Error(`Assertion failed: ${message}`) +} + +function requireStructuredContent(result: unknown, label: string): Record { + if (!result || typeof result !== 'object') + throw new Error(`${label}: result is not an object`) + + const sc = (result as { structuredContent?: unknown }).structuredContent + if (!sc || typeof sc !== 'object') + throw new Error(`${label}: missing structuredContent`) + + return sc as Record +} + +function createProjectDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'e2e-terminal-exec-')) + writeFileSync(join(dir, 'README.md'), '# e2e terminal exec test\n', 'utf8') + writeFileSync(join(dir, 'index.ts'), 'export const ok = true\n', 'utf8') + return dir +} + +async function createClient(): Promise { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = (env.COMPUTER_USE_SMOKE_SERVER_ARGS || 'start').split(/\s+/).filter(Boolean) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + // Desktop is dry-run, but terminal runner is REAL + COMPUTER_USE_EXECUTOR: 'dry-run', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_SESSION_TAG: 'e2e-terminal-exec', + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,1920,1080', + COMPUTER_USE_OPENABLE_APPS: 'Finder,Terminal,Visual Studio Code', + }, + stderr: 'pipe', + }) + + const client = new Client({ + name: '@proj-airi/e2e-terminal-exec', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[stderr] ${text}`) + }) + + await client.connect(transport) + return client +} + +// --------------------------------------------------------------------------- +// Test phases +// --------------------------------------------------------------------------- + +async function phase1_validateWorkspace(client: Client, projectPath: string) { + console.info('\n── Phase 1: workflow_validate_workspace with real commands ──') + + const result = await client.callTool({ + name: 'workflow_validate_workspace', + arguments: { + projectPath, + ideApp: 'Visual Studio Code', + changesCommand: 'echo "M index.ts"', + checkCommand: 'echo "all checks passed"', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_validate_workspace') + console.info(` Status: ${data.status}`) + assert( + data.status === 'completed', + `expected completed, got ${String(data.status)}`, + ) + + const steps = data.stepResults as Array<{ label: string, succeeded: boolean, status: string }> + for (const s of steps) { + console.info(` ${s.succeeded ? '✓' : '✗'} ${s.label} (${s.status})`) + } + + // Verify the terminal exec steps ran real commands + const pwdStep = steps.find(s => s.label === 'Confirm project working directory') + assert(pwdStep?.succeeded === true, 'pwd step must succeed') + + const changesStep = steps.find(s => s.label === 'Inspect local changes') + assert(changesStep?.succeeded === true, 'changes step must succeed') + + const checkStep = steps.find(s => s.label === 'Run workspace validation') + assert(checkStep?.succeeded === true, 'check step must succeed') + + return data +} + +async function phase2_verifyTerminalState(client: Client, projectPath: string) { + console.info('\n── Phase 2: Verify terminal state reflects exec chain ──') + + const result = await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'desktop_get_state') + assert(data.status === 'ok', `get_state: expected ok, got ${data.status}`) + + const runState = data.runState as Record + const terminalState = runState.terminalState as Record | undefined + + console.info(` Terminal state: ${JSON.stringify(terminalState)}`) + + assert(terminalState != null, 'terminalState must be present') + assert( + terminalState.effectiveCwd === projectPath, + `expected effectiveCwd=${projectPath}, got ${String(terminalState.effectiveCwd)}`, + ) + assert( + terminalState.lastExitCode === 0, + `expected lastExitCode=0, got ${String(terminalState.lastExitCode)}`, + ) + assert( + typeof terminalState.lastCommandSummary === 'string' + && terminalState.lastCommandSummary.includes('all checks passed'), + `expected lastCommandSummary to include "all checks passed", got ${String(terminalState.lastCommandSummary)}`, + ) + console.info(' ✓ Terminal state is correct after exec chain') + return data +} + +async function phase3_runTests(client: Client, projectPath: string) { + console.info('\n── Phase 3: workflow_run_tests to prove continuation ──') + + const result = await client.callTool({ + name: 'workflow_run_tests', + arguments: { + projectPath, + testCommand: 'echo "test suite passed"', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_run_tests') + console.info(` Status: ${data.status}`) + assert(data.status === 'completed', `expected completed, got ${String(data.status)}`) + + const steps = data.stepResults as Array<{ label: string, succeeded: boolean }> + for (const s of steps) { + console.info(` ${s.succeeded ? '✓' : '✗'} ${s.label}`) + } + + return data +} + +async function phase4_finalState(client: Client, _projectPath: string) { + console.info('\n── Phase 4: Final terminal state after full chain ──') + + const result = await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'desktop_get_state') + const runState = data.runState as Record + const terminalState = runState.terminalState as Record | undefined + + assert(terminalState != null, 'terminalState must be present after full chain') + assert( + terminalState.lastExitCode === 0, + `expected lastExitCode=0 after tests, got ${String(terminalState.lastExitCode)}`, + ) + assert( + typeof terminalState.lastCommandSummary === 'string' + && terminalState.lastCommandSummary.includes('test suite passed'), + `expected lastCommandSummary to include "test suite passed", got ${String(terminalState.lastCommandSummary)}`, + ) + console.info(' ✓ Terminal state is correct after full chain') + return data +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.info('╔═══════════════════════════════════════════════════════╗') + console.info('║ E2E Release Gate: Terminal Exec Happy Path ║') + console.info('╚═══════════════════════════════════════════════════════╝') + + const projectPath = createProjectDir() + console.info(` Project directory: ${projectPath}`) + + const client = await createClient() + + try { + const { tools } = await client.listTools() + const names = new Set(tools.map(t => t.name)) + for (const t of ['workflow_validate_workspace', 'workflow_run_tests', 'desktop_get_state']) { + assert(names.has(t), `missing required tool: ${t}`) + } + console.info(` ${tools.length} tools available`) + + await phase1_validateWorkspace(client, projectPath) + await phase2_verifyTerminalState(client, projectPath) + await phase3_runTests(client, projectPath) + await phase4_finalState(client, projectPath) + + console.info('\n╔═══════════════════════════════════════════════════════╗') + console.info('║ TERMINAL EXEC E2E — ALL PHASES PASSED ║') + console.info('╚═══════════════════════════════════════════════════════╝') + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error('\n❌ TERMINAL EXEC E2E FAILED') + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-terminal-pty.ts b/services/computer-use-mcp/src/bin/e2e-terminal-pty.ts new file mode 100644 index 000000000..188374cce --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-terminal-pty.ts @@ -0,0 +1,304 @@ +/** + * Real E2E: Terminal PTY happy path. + * + * Proves the PTY surface works end-to-end through a real MCP stdio + * transport with real `node-pty`: + * + * 1. pty_create → allocates a real pseudo-terminal + * 2. Run the deterministic interactive-echo fixture + * 3. pty_read_screen → read real terminal buffer + * 4. pty_send_input → write real keystrokes + * 5. pty_read_screen → verify echo output + * 6. pty_destroy → verify cleanup + * + * No mocks. The PTY session is a real pseudo-terminal running a real + * Node.js process on the host machine. + * + * Usage: + * pnpm -F @proj-airi/computer-use-mcp e2e:terminal-pty + */ + +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const fixtureScript = resolve(packageDir, 'fixtures/interactive-echo.mjs') + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assert(condition: boolean, message: string): asserts condition { + if (!condition) + throw new Error(`Assertion failed: ${message}`) +} + +function requireStructuredContent(result: unknown, label: string): Record { + if (!result || typeof result !== 'object') + throw new Error(`${label}: result is not an object`) + + const sc = (result as { structuredContent?: unknown }).structuredContent + if (!sc || typeof sc !== 'object') + throw new Error(`${label}: missing structuredContent`) + + return sc as Record +} + +async function createClient(): Promise { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = (env.COMPUTER_USE_SMOKE_SERVER_ARGS || 'start').split(/\s+/).filter(Boolean) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'dry-run', + // Approval disabled — proves the PTY lifecycle works without + // the extra approval ceremony. A separate E2E can test approval. + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_SESSION_TAG: 'e2e-terminal-pty', + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,1920,1080', + COMPUTER_USE_OPENABLE_APPS: 'Finder,Terminal', + }, + stderr: 'pipe', + }) + + const client = new Client({ + name: '@proj-airi/e2e-terminal-pty', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[stderr] ${text}`) + }) + + await client.connect(transport) + return client +} + +// --------------------------------------------------------------------------- +// Test phases +// --------------------------------------------------------------------------- + +async function phase1_checkPtyAvailable(client: Client) { + console.info('\n── Phase 1: pty_get_status → verify PTY support ──') + + const result = await client.callTool({ + name: 'pty_get_status', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'pty_get_status') + console.info(` PTY available: ${data.ptyAvailable}`) + assert(data.ptyAvailable === true, 'node-pty must be available for this E2E') + assert(data.status === 'ok', `expected ok, got ${String(data.status)}`) + + const sessions = data.sessions as unknown[] + console.info(` Active sessions: ${sessions.length}`) + assert(sessions.length === 0, 'should start with zero PTY sessions') +} + +async function phase2_createPtyAndRunFixture(client: Client): Promise { + console.info('\n── Phase 2: pty_create → allocate real PTY ──') + + const result = await client.callTool({ + name: 'pty_create', + arguments: { + rows: 24, + cols: 80, + cwd: packageDir, + }, + }) + + const data = requireStructuredContent(result, 'pty_create') + console.info(` Status: ${data.status}`) + assert(data.status === 'ok', `expected ok, got ${String(data.status)}`) + + const session = data.session as Record + const sessionId = String(session.id) + console.info(` Session: ${sessionId} (pid ${session.pid}, alive: ${session.alive})`) + assert(session.alive === true, 'session must be alive') + assert(typeof session.pid === 'number', 'session must have a real pid') + + // Send the command to run the interactive fixture + console.info(' Launching interactive-echo fixture...') + const sendResult = await client.callTool({ + name: 'pty_send_input', + arguments: { + sessionId, + data: `node ${fixtureScript}\r`, + }, + }) + const sendData = requireStructuredContent(sendResult, 'pty_send_input') + assert(sendData.status === 'ok', `send_input: expected ok, got ${String(sendData.status)}`) + + // Wait for the fixture to start (zsh init + Node.js startup) + await delay(3000) + + return sessionId +} + +async function phase3_readScreen(client: Client, sessionId: string) { + console.info('\n── Phase 3: pty_read_screen → verify fixture started ──') + + const result = await client.callTool({ + name: 'pty_read_screen', + arguments: { sessionId }, + }) + + const data = requireStructuredContent(result, 'pty_read_screen') + assert(data.status === 'ok', `read_screen: expected ok, got ${String(data.status)}`) + + const screenContent = String(data.screenContent ?? '') + console.info(` Screen content:\n${screenContent.split('\n').map(l => ` | ${l}`).join('\n')}`) + assert( + screenContent.includes('READY>'), + `expected to see "READY>" prompt, got: ${screenContent.slice(0, 200)}`, + ) + console.info(' ✓ Fixture is running and waiting for input') +} + +async function phase4_sendInputAndVerify(client: Client, sessionId: string) { + console.info('\n── Phase 4: pty_send_input → send "hello e2e" ──') + + const result = await client.callTool({ + name: 'pty_send_input', + arguments: { + sessionId, + data: 'hello e2e\r', + }, + }) + + const data = requireStructuredContent(result, 'pty_send_input') + assert(data.status === 'ok', `send_input: expected ok, got ${String(data.status)}`) + console.info(` Wrote ${data.bytesWritten} bytes`) + + // Wait for the fixture to process + await delay(500) + + console.info(' Reading screen after input...') + const readResult = await client.callTool({ + name: 'pty_read_screen', + arguments: { sessionId }, + }) + + const readData = requireStructuredContent(readResult, 'pty_read_screen') + const screenContent = String(readData.screenContent ?? '') + console.info(` Screen content:\n${screenContent.split('\n').map(l => ` | ${l}`).join('\n')}`) + + assert( + screenContent.includes('ECHO: hello e2e'), + `expected to see "ECHO: hello e2e", got: ${screenContent.slice(0, 300)}`, + ) + assert( + screenContent.includes('DONE'), + `expected to see "DONE", got: ${screenContent.slice(0, 300)}`, + ) + console.info(' ✓ Interactive fixture echoed input correctly') +} + +async function phase5_verifyState(client: Client, sessionId: string) { + console.info('\n── Phase 5: desktop_get_state → verify PTY state ──') + + const result = await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'desktop_get_state') + const runState = data.runState as Record + const ptySessions = runState.ptySessions as Array> | undefined + + assert(ptySessions != null, 'ptySessions must be present') + const ourSession = ptySessions.find(s => s.id === sessionId) + assert(ourSession != null, `session ${sessionId} must be in state`) + console.info(` Session in state: ${JSON.stringify(ourSession)}`) + + // Verify audit log + const ptyAuditLog = runState.ptyAuditLog as Array> | undefined + assert(ptyAuditLog != null, 'ptyAuditLog must be present') + console.info(` Audit entries: ${ptyAuditLog.length}`) + assert(ptyAuditLog.length >= 3, `expected ≥3 audit entries (create, read, send, read), got ${ptyAuditLog.length}`) + + const events = ptyAuditLog.map(e => e.event) + console.info(` Audit events: ${events.join(', ')}`) + assert(events.includes('create'), 'audit must include create') + assert(events.includes('read_screen'), 'audit must include read_screen') + assert(events.includes('send_input'), 'audit must include send_input') + console.info(' ✓ PTY state and audit log are correct') +} + +async function phase6_destroyAndVerify(client: Client, sessionId: string) { + console.info('\n── Phase 6: pty_destroy → cleanup ──') + + const result = await client.callTool({ + name: 'pty_destroy', + arguments: { sessionId }, + }) + + const data = requireStructuredContent(result, 'pty_destroy') + assert(data.status === 'ok', `destroy: expected ok, got ${String(data.status)}`) + console.info(` Destroyed: ${sessionId}`) + + // Verify session is gone + const statusResult = await client.callTool({ + name: 'pty_get_status', + arguments: {}, + }) + + const statusData = requireStructuredContent(statusResult, 'pty_get_status') + const sessions = statusData.sessions as unknown[] + assert(sessions.length === 0, `expected 0 sessions after destroy, got ${sessions.length}`) + console.info(' ✓ Session cleaned up') +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.info('╔═══════════════════════════════════════════════════════╗') + console.info('║ E2E Release Gate: Terminal PTY Happy Path ║') + console.info('╚═══════════════════════════════════════════════════════╝') + + const client = await createClient() + + try { + const { tools } = await client.listTools() + const names = new Set(tools.map(t => t.name)) + for (const t of ['pty_get_status', 'pty_create', 'pty_send_input', 'pty_read_screen', 'pty_destroy', 'desktop_get_state']) { + assert(names.has(t), `missing required tool: ${t}`) + } + console.info(` ${tools.length} tools available`) + + await phase1_checkPtyAvailable(client) + const sessionId = await phase2_createPtyAndRunFixture(client) + await phase3_readScreen(client, sessionId) + await phase4_sendInputAndVerify(client, sessionId) + await phase5_verifyState(client, sessionId) + await phase6_destroyAndVerify(client, sessionId) + + console.info('\n╔═══════════════════════════════════════════════════════╗') + console.info('║ TERMINAL PTY E2E — ALL PHASES PASSED ║') + console.info('╚═══════════════════════════════════════════════════════╝') + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error('\n❌ TERMINAL PTY E2E FAILED') + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/e2e-terminal-self-acquire.ts b/services/computer-use-mcp/src/bin/e2e-terminal-self-acquire.ts new file mode 100644 index 000000000..9b5629755 --- /dev/null +++ b/services/computer-use-mcp/src/bin/e2e-terminal-self-acquire.ts @@ -0,0 +1,289 @@ +/** + * Real E2E: Terminal Lane v2 — PTY self-acquire happy path. + * + * The most valuable release gate — proves in a single pass: + * + * 1. Surface resolver detects an interactive command + * 2. Workflow engine self-acquires a PTY through the unified approval path + * 3. Engine executes the command on the acquired PTY + * 4. Step succeeds without outward reroute + * 5. State is consistent (bindings, audit, surface decisions) + * + * Scenario: + * - Call workflow_validate_workspace with `vim --version` as checkCommand + * - Early steps (pwd, git diff) succeed via terminal_exec (auto_default_exec) + * - "Run workspace validation" step surface-resolves to auto_interactive_command + * - Engine self-acquires a real PTY, sends the command, reads screen output + * - Step succeeds — no reroute + * - Verify state consistency (bindings, audit, surface decisions, PTY session) + * + * NOTE: No pre-created PTY. The workflow self-acquires. + * + * Usage: + * pnpm -F @proj-airi/computer-use-mcp e2e:terminal-self-acquire + */ + +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assert(condition: boolean, message: string): asserts condition { + if (!condition) + throw new Error(`Assertion failed: ${message}`) +} + +function requireStructuredContent(result: unknown, label: string): Record { + if (!result || typeof result !== 'object') + throw new Error(`${label}: result is not an object`) + + const sc = (result as { structuredContent?: unknown }).structuredContent + if (!sc || typeof sc !== 'object') + throw new Error(`${label}: missing structuredContent`) + + return sc as Record +} + +function createProjectDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'e2e-terminal-self-acquire-')) + writeFileSync(join(dir, 'README.md'), '# e2e terminal self-acquire test\n', 'utf8') + writeFileSync(join(dir, 'index.ts'), 'export const ok = true\n', 'utf8') + return dir +} + +async function createClient(): Promise { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = (env.COMPUTER_USE_SMOKE_SERVER_ARGS || 'start').split(/\s+/).filter(Boolean) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'dry-run', + COMPUTER_USE_APPROVAL_MODE: 'never', + COMPUTER_USE_SESSION_TAG: 'e2e-terminal-self-acquire', + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,1920,1080', + COMPUTER_USE_OPENABLE_APPS: 'Finder,Terminal,Visual Studio Code', + }, + stderr: 'pipe', + }) + + const client = new Client({ + name: '@proj-airi/e2e-terminal-self-acquire', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[stderr] ${text}`) + }) + + await client.connect(transport) + return client +} + +// --------------------------------------------------------------------------- +// Test phases +// --------------------------------------------------------------------------- + +async function phase1_selfAcquirePty(client: Client, projectPath: string) { + console.info('\n── Phase 1: workflow_validate_workspace with interactive checkCommand ──') + console.info(' No pre-created PTY. The engine self-acquires.') + + const result = await client.callTool({ + name: 'workflow_validate_workspace', + arguments: { + projectPath, + ideApp: 'Visual Studio Code', + changesCommand: 'echo "M index.ts"', + // `vim --version` matches `^vim\b` → auto_interactive_command → PTY self-acquire + checkCommand: 'vim --version', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_validate_workspace') + console.info(` Kind: ${data.kind}`) + console.info(` Status: ${data.status}`) + + // v2: workflow should complete (not reroute) because it self-acquires PTY + const steps = data.stepResults as Array<{ + label: string + succeeded: boolean + status: string + explanation?: string + }> + for (const s of steps) { + console.info(` ${s.succeeded ? '✓' : '✗'} ${s.label} (${s.status})`) + } + + // Early steps should succeed via exec + const pwdStep = steps.find(s => s.label === 'Confirm project working directory') + assert(pwdStep?.succeeded === true, 'pwd step must have succeeded via exec') + + const changesStep = steps.find(s => s.label === 'Inspect local changes') + assert(changesStep?.succeeded === true, 'changes step must have succeeded via exec') + + // The validation step should succeed via PTY self-acquire + const validationStep = steps.find(s => s.label === 'Run workspace validation') + assert( + validationStep?.succeeded === true, + `validation step must have succeeded via PTY self-acquire, got status=${validationStep?.status}`, + ) + assert( + validationStep?.explanation?.includes('PTY') === true, + `explanation must mention PTY, got: ${validationStep?.explanation}`, + ) + console.info(` ✓ Validation step succeeded via PTY: ${validationStep?.explanation}`) + + return data +} + +async function phase2_verifyState(client: Client) { + console.info('\n── Phase 2: Verify state consistency ──') + + const result = await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'desktop_get_state') + const runState = data.runState as Record + + // PTY sessions — one should have been self-acquired + const ptySessions = runState.ptySessions as Array> | undefined + if (ptySessions && ptySessions.length > 0) { + console.info(` PTY sessions: ${ptySessions.length}`) + for (const s of ptySessions) { + console.info(` ${s.id} (alive=${s.alive})`) + } + } + else { + console.info(' PTY sessions: (may have been cleaned up after step)') + } + + // Surface decisions — should include a 'pty' decision from the surface resolver + const surfaceDecisions = runState.surfaceDecisions as Array> | undefined + if (surfaceDecisions && surfaceDecisions.length > 0) { + const ptyDecision = surfaceDecisions.find(d => d.surface === 'pty') + console.info(` PTY surface decision: ${JSON.stringify(ptyDecision)}`) + assert(ptyDecision != null, 'must have a pty surface decision from self-acquire') + assert( + typeof ptyDecision.reason === 'string' && ptyDecision.reason.length > 0, + 'surface decision must have a reason', + ) + } + + // Audit log + const auditLog = runState.ptyAuditLog as Array> | undefined + if (auditLog) { + const events = auditLog.map(e => e.event) + console.info(` Audit events: ${events.join(', ')}`) + assert(events.includes('create'), 'audit must include a create event from self-acquire') + } + + // Step bindings + const stepBindings = runState.stepTerminalBindings as Array> | undefined + if (stepBindings && stepBindings.length > 0) { + const ptyBinding = stepBindings.find(b => b.surface === 'pty') + console.info(` PTY step binding: ${JSON.stringify(ptyBinding)}`) + assert(ptyBinding != null, 'must have a pty step binding from self-acquire') + assert( + typeof ptyBinding.ptySessionId === 'string', + 'binding must have ptySessionId', + ) + } + + console.info(' ✓ State is consistent with PTY self-acquire') +} + +async function phase3_cleanup(client: Client) { + console.info('\n── Phase 3: Cleanup ──') + + // Get any active PTY sessions and destroy them + const statusResult = await client.callTool({ + name: 'pty_get_status', + arguments: {}, + }) + + const statusData = requireStructuredContent(statusResult, 'pty_get_status') + const sessions = statusData.sessions as Array> | undefined + + if (sessions && sessions.length > 0) { + for (const s of sessions) { + const sessionId = String(s.id) + const destroyResult = await client.callTool({ + name: 'pty_destroy', + arguments: { sessionId }, + }) + const destroyData = requireStructuredContent(destroyResult, 'pty_destroy') + console.info(` Destroyed ${sessionId}: ${destroyData.status}`) + } + } + else { + console.info(' No PTY sessions to clean up') + } + + console.info(' ✓ Cleanup complete') +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.info('╔══════════════════════════════════════════════════════════╗') + console.info('║ E2E Release Gate: Terminal Lane v2 — PTY Self-Acquire ║') + console.info('╚══════════════════════════════════════════════════════════╝') + + const projectPath = createProjectDir() + console.info(` Project directory: ${projectPath}`) + + const client = await createClient() + + try { + const { tools } = await client.listTools() + const names = new Set(tools.map(t => t.name)) + for (const t of [ + 'workflow_validate_workspace', + 'pty_get_status', + 'pty_destroy', + 'desktop_get_state', + ]) { + assert(names.has(t), `missing required tool: ${t}`) + } + console.info(` ${tools.length} tools available`) + + // The core flow — no pre-created PTY + await phase1_selfAcquirePty(client, projectPath) + await phase2_verifyState(client) + await phase3_cleanup(client) + + console.info('\n╔══════════════════════════════════════════════════════════╗') + console.info('║ PTY SELF-ACQUIRE E2E — ALL PHASES PASSED ║') + console.info('╚══════════════════════════════════════════════════════════╝') + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error('\n❌ PTY SELF-ACQUIRE E2E FAILED') + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/run.ts b/services/computer-use-mcp/src/bin/run.ts new file mode 100644 index 000000000..9a74300b7 --- /dev/null +++ b/services/computer-use-mcp/src/bin/run.ts @@ -0,0 +1,7 @@ +import { startComputerUseMcpServer } from '../server' + +async function main() { + await startComputerUseMcpServer() +} + +void main() diff --git a/services/computer-use-mcp/src/bin/runner.ts b/services/computer-use-mcp/src/bin/runner.ts new file mode 100644 index 000000000..305e8d8e9 --- /dev/null +++ b/services/computer-use-mcp/src/bin/runner.ts @@ -0,0 +1,167 @@ +import type { RunnerRequest, RunnerResponse } from '../runner/protocol' + +import { createInterface } from 'node:readline' + +import { LinuxX11RunnerService } from '../runner/service' + +const runner = new LinuxX11RunnerService() +const rl = createInterface({ + input: process.stdin, + crlfDelay: Infinity, +}) + +let queue = Promise.resolve() + +async function writeResponse(response: RunnerResponse) { + process.stdout.write(`${JSON.stringify(response)}\n`) +} + +async function handleRequest(request: RunnerRequest) { + try { + switch (request.method) { + case 'initialize': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.initialize(request.params as never), + }) + return + case 'getExecutionTarget': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.getExecutionTarget(), + }) + return + case 'getDisplayInfo': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.getDisplayInfo(), + }) + return + case 'getForegroundContext': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.getForegroundContext(), + }) + return + case 'getPermissionInfo': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.getPermissionInfo(), + }) + return + case 'takeScreenshot': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.takeScreenshot(request.params as never), + }) + return + case 'click': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.click(request.params as never), + }) + return + case 'typeText': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.typeText(request.params as never), + }) + return + case 'pressKeys': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.pressKeys(request.params as never), + }) + return + case 'scroll': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.scroll(request.params as never), + }) + return + case 'wait': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.wait(request.params as never), + }) + return + case 'openTestTarget': + await writeResponse({ + id: request.id, + ok: true, + result: await runner.openTestTarget(), + }) + return + case 'shutdown': + await runner.shutdown() + await writeResponse({ + id: request.id, + ok: true, + result: { ok: true }, + }) + process.exit(0) + } + } + catch (error) { + await writeResponse({ + id: request.id, + ok: false, + error: { + message: error instanceof Error ? error.message : String(error), + }, + }) + } +} + +function enqueueRequest(request: RunnerRequest) { + queue = queue.then(async () => { + await handleRequest(request) + }).catch(async (error) => { + await writeResponse({ + id: request.id, + ok: false, + error: { + message: error instanceof Error ? error.message : String(error), + }, + }) + }) +} + +rl.on('line', (line) => { + const trimmed = line.trim() + if (!trimmed) + return + + try { + const request = JSON.parse(trimmed) as RunnerRequest + enqueueRequest(request) + } + catch (error) { + process.stderr.write(`invalid runner request: ${error instanceof Error ? error.message : String(error)}\n`) + } +}) + +async function shutdown() { + await runner.shutdown().catch(() => {}) +} + +process.on('SIGINT', () => { + void shutdown().finally(() => process.exit(0)) +}) +process.on('SIGTERM', () => { + void shutdown().finally(() => process.exit(0)) +}) +process.stdin.on('end', () => { + void shutdown().finally(() => process.exit(0)) +}) diff --git a/services/computer-use-mcp/src/bin/smoke-macos.ts b/services/computer-use-mcp/src/bin/smoke-macos.ts new file mode 100644 index 000000000..959036a1c --- /dev/null +++ b/services/computer-use-mcp/src/bin/smoke-macos.ts @@ -0,0 +1,142 @@ +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +function parseCommandArgs(raw: string | undefined, fallback: string[]) { + if (!raw?.trim()) + return fallback + + return raw + .split(/\s+/) + .map(item => item.trim()) + .filter(Boolean) +} + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') + throw new Error(`${label} did not return an object result`) + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') + throw new Error(`${label} missing structuredContent`) + + return structuredContent as Record +} + +async function approveFirstPending(client: Client, expectedToolName: string) { + const pending = await client.callTool({ + name: 'desktop_list_pending_actions', + arguments: {}, + }) + const pendingData = requireStructuredContent(pending, 'desktop_list_pending_actions') + const pendingActions = Array.isArray(pendingData.pendingActions) ? pendingData.pendingActions : [] + const first = pendingActions[0] as Record | undefined + if (!first) + throw new Error(`no pending action after ${expectedToolName}`) + + const pendingId = String(first.id || '') + if (!pendingId) + throw new Error(`pending action missing id after ${expectedToolName}`) + + const approved = await client.callTool({ + name: 'desktop_approve_pending_action', + arguments: { id: pendingId }, + }) + return requireStructuredContent(approved, 'desktop_approve_pending_action') +} + +async function main() { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start']) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: env.COMPUTER_USE_SMOKE_EXECUTOR || 'macos-local', + COMPUTER_USE_APPROVAL_MODE: env.COMPUTER_USE_SMOKE_APPROVAL_MODE || 'actions', + COMPUTER_USE_OPENABLE_APPS: env.COMPUTER_USE_OPENABLE_APPS || 'Terminal,Cursor,Google Chrome', + }, + stderr: 'pipe', + }) + const client = new Client({ + name: '@proj-airi/computer-use-mcp-smoke-macos', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[computer-use-mcp stderr] ${text}`) + }) + + try { + await client.connect(transport) + + const capabilities = await client.callTool({ + name: 'desktop_get_capabilities', + arguments: {}, + }) + const capabilitiesData = requireStructuredContent(capabilities, 'desktop_get_capabilities') + const executionTarget = capabilitiesData.executionTarget as Record | undefined + if (executionTarget?.mode !== 'local-windowed') { + throw new Error(`desktop_get_capabilities expected local-windowed target, got ${String(executionTarget?.mode)}`) + } + + const observation = await client.callTool({ + name: 'desktop_observe_windows', + arguments: { limit: 8 }, + }) + const observationData = requireStructuredContent(observation, 'desktop_observe_windows') + + const openTerminal = await client.callTool({ + name: 'desktop_open_app', + arguments: { app: 'Terminal' }, + }) + const openTerminalData = requireStructuredContent(openTerminal, 'desktop_open_app') + if (openTerminalData.status !== 'approval_required') + throw new Error(`desktop_open_app expected approval_required, got ${String(openTerminalData.status)}`) + const approvedOpen = await approveFirstPending(client, 'desktop_open_app') + + const terminalExec = await client.callTool({ + name: 'terminal_exec', + arguments: { command: 'pwd' }, + }) + const terminalExecData = requireStructuredContent(terminalExec, 'terminal_exec') + if (terminalExecData.status !== 'approval_required') + throw new Error(`terminal_exec expected approval_required, got ${String(terminalExecData.status)}`) + const approvedExec = await approveFirstPending(client, 'terminal_exec') + + const terminalState = await client.callTool({ + name: 'terminal_get_state', + arguments: {}, + }) + + console.info(JSON.stringify({ + ok: true, + verified: { + executionTarget, + observation: observationData.backendResult || observationData, + approvedOpen, + approvedExec, + terminalState: requireStructuredContent(terminalState, 'terminal_get_state').terminalState, + }, + }, null, 2)) + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/smoke-remote.ts b/services/computer-use-mcp/src/bin/smoke-remote.ts new file mode 100644 index 000000000..071bced91 --- /dev/null +++ b/services/computer-use-mcp/src/bin/smoke-remote.ts @@ -0,0 +1,227 @@ +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +function parseCommandArgs(raw: string | undefined, fallback: string[]) { + if (!raw?.trim()) { + return fallback + } + + return raw + .split(/\s+/) + .map(item => item.trim()) + .filter(Boolean) +} + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') { + throw new Error(`${label} did not return an object result`) + } + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') { + throw new Error(`${label} missing structuredContent`) + } + + return structuredContent as Record +} + +async function approveFirstPending(client: Client, label: string) { + const pending = await client.callTool({ + name: 'desktop_list_pending_actions', + arguments: {}, + }) + const pendingData = requireStructuredContent(pending, label) + const pendingActions = Array.isArray(pendingData.pendingActions) ? pendingData.pendingActions : [] + if (pendingActions.length === 0) { + throw new Error(`${label} returned no pending actions`) + } + + const id = String((pendingActions[0] as Record).id || '') + if (!id) { + throw new Error(`${label} missing pending action id`) + } + + const approved = await client.callTool({ + name: 'desktop_approve_pending_action', + arguments: { id }, + }) + + return { + id, + approved: requireStructuredContent(approved, 'desktop_approve_pending_action'), + } +} + +async function main() { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start']) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'linux-x11', + COMPUTER_USE_APPROVAL_MODE: env.COMPUTER_USE_SMOKE_APPROVAL_MODE || 'actions', + COMPUTER_USE_SESSION_TAG: env.COMPUTER_USE_SMOKE_SESSION_TAG || 'azure-remote-smoke', + COMPUTER_USE_ALLOWED_BOUNDS: env.COMPUTER_USE_SMOKE_ALLOWED_BOUNDS || '0,0,1280,720', + COMPUTER_USE_ENABLE_TEST_TOOLS: 'true', + }, + stderr: 'pipe', + }) + const client = new Client({ + name: '@proj-airi/computer-use-mcp-remote-smoke', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) { + console.error(`[computer-use-mcp stderr] ${text}`) + } + }) + + try { + await client.connect(transport) + + const tools = await client.listTools() + const toolNames = new Set(tools.tools.map(tool => tool.name)) + for (const required of [ + 'desktop_get_capabilities', + 'desktop_open_test_target', + 'desktop_screenshot', + 'desktop_click', + 'desktop_type_text', + 'desktop_wait', + 'desktop_list_pending_actions', + 'desktop_approve_pending_action', + ]) { + if (!toolNames.has(required)) { + throw new Error(`missing required tool: ${required}`) + } + } + + const capabilities = await client.callTool({ + name: 'desktop_get_capabilities', + arguments: {}, + }) + const capabilitiesData = requireStructuredContent(capabilities, 'desktop_get_capabilities') + const executionTarget = capabilitiesData.executionTarget as Record | undefined + if (!executionTarget || executionTarget.mode !== 'remote') { + throw new Error('desktop_get_capabilities did not report a remote execution target') + } + + const opened = await client.callTool({ + name: 'desktop_open_test_target', + arguments: {}, + }) + const openedData = requireStructuredContent(opened, 'desktop_open_test_target') + const point = openedData.recommendedClickPoint as Record | undefined + if (!point || typeof point.x !== 'number' || typeof point.y !== 'number') { + throw new Error('desktop_open_test_target did not return a recommendedClickPoint') + } + + const screenshotBefore = await client.callTool({ + name: 'desktop_screenshot', + arguments: { + label: 'remote-smoke-before', + }, + }) + const screenshotBeforeData = requireStructuredContent(screenshotBefore, 'desktop_screenshot before') + + const click = await client.callTool({ + name: 'desktop_click', + arguments: { + x: point.x, + y: point.y, + captureAfter: false, + }, + }) + const clickData = requireStructuredContent(click, 'desktop_click') + if (clickData.status !== 'approval_required') { + throw new Error(`desktop_click expected approval_required, got ${String(clickData.status)}`) + } + + const approvedClick = await approveFirstPending(client, 'desktop_list_pending_actions after click') + if (approvedClick.approved.status !== 'executed') { + throw new Error(`desktop_approve_pending_action for click expected executed, got ${String(approvedClick.approved.status)}`) + } + + const typeText = await client.callTool({ + name: 'desktop_type_text', + arguments: { + text: 'AIRI remote linux-x11 smoke', + pressEnter: false, + captureAfter: true, + }, + }) + const typeTextData = requireStructuredContent(typeText, 'desktop_type_text') + if (typeTextData.status !== 'approval_required') { + throw new Error(`desktop_type_text expected approval_required, got ${String(typeTextData.status)}`) + } + + const approvedTypeText = await approveFirstPending(client, 'desktop_list_pending_actions after type_text') + if (approvedTypeText.approved.status !== 'executed') { + throw new Error(`desktop_approve_pending_action for type_text expected executed, got ${String(approvedTypeText.approved.status)}`) + } + + const waited = await client.callTool({ + name: 'desktop_wait', + arguments: { + durationMs: 500, + }, + }) + const waitedData = requireStructuredContent(waited, 'desktop_wait') + if (waitedData.status !== 'executed') { + throw new Error(`desktop_wait expected executed, got ${String(waitedData.status)}`) + } + + const screenshotAfter = await client.callTool({ + name: 'desktop_screenshot', + arguments: { + label: 'remote-smoke-after', + }, + }) + const screenshotAfterData = requireStructuredContent(screenshotAfter, 'desktop_screenshot after') + + console.info(JSON.stringify({ + ok: true, + verified: { + toolCount: tools.tools.length, + executionTarget, + openedTarget: { + appName: openedData.appName, + windowTitle: openedData.windowTitle, + recommendedClickPoint: openedData.recommendedClickPoint, + }, + screenshotBefore: screenshotBeforeData.screenshot, + approvedClick: { + id: approvedClick.id, + status: approvedClick.approved.status, + }, + approvedTypeText: { + id: approvedTypeText.id, + status: approvedTypeText.approved.status, + }, + waited: waitedData.status, + screenshotAfter: screenshotAfterData.screenshot, + }, + }, null, 2)) + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/smoke-stdio.ts b/services/computer-use-mcp/src/bin/smoke-stdio.ts new file mode 100644 index 000000000..250d2e619 --- /dev/null +++ b/services/computer-use-mcp/src/bin/smoke-stdio.ts @@ -0,0 +1,206 @@ +import { dirname, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +function parseCommandArgs(raw: string | undefined, fallback: string[]) { + if (!raw?.trim()) { + return fallback + } + + return raw + .split(/\s+/) + .map(item => item.trim()) + .filter(Boolean) +} + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') { + throw new Error(`${label} did not return an object result`) + } + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') { + throw new Error(`${label} missing structuredContent`) + } + + return structuredContent as Record +} + +function hasImageContent(result: unknown) { + if (!result || typeof result !== 'object') { + return false + } + + const content = (result as { content?: unknown }).content + if (!Array.isArray(content)) { + return false + } + + return content.some((item) => { + if (!item || typeof item !== 'object') { + return false + } + + const record = item as Record + return record.type === 'image' + && typeof record.data === 'string' + && typeof record.mimeType === 'string' + }) +} + +async function main() { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start']) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: env.COMPUTER_USE_SMOKE_EXECUTOR || 'dry-run', + COMPUTER_USE_APPROVAL_MODE: env.COMPUTER_USE_SMOKE_APPROVAL_MODE || 'actions', + COMPUTER_USE_SESSION_TAG: env.COMPUTER_USE_SMOKE_SESSION_TAG || 'smoke-standalone', + COMPUTER_USE_ALLOWED_BOUNDS: env.COMPUTER_USE_SMOKE_ALLOWED_BOUNDS || '0,0,1280,800', + }, + stderr: 'pipe', + }) + const client = new Client({ + name: '@proj-airi/computer-use-mcp-smoke', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) { + console.error(`[computer-use-mcp stderr] ${text}`) + } + }) + + try { + await client.connect(transport) + + const tools = await client.listTools() + const toolNames = new Set(tools.tools.map(tool => tool.name)) + for (const required of [ + 'desktop_get_capabilities', + 'desktop_screenshot', + 'desktop_click', + 'desktop_list_pending_actions', + 'desktop_approve_pending_action', + ]) { + if (!toolNames.has(required)) { + throw new Error(`missing required tool: ${required}`) + } + } + + const capabilities = await client.callTool({ + name: 'desktop_get_capabilities', + arguments: {}, + }) + const capabilitiesData = requireStructuredContent(capabilities, 'desktop_get_capabilities') + if (typeof capabilitiesData.launchContext !== 'object' || capabilitiesData.launchContext == null) { + throw new Error('desktop_get_capabilities missing launchContext') + } + if (typeof capabilitiesData.displayInfo !== 'object' || capabilitiesData.displayInfo == null) { + throw new Error('desktop_get_capabilities missing displayInfo') + } + + const screenshot = await client.callTool({ + name: 'desktop_screenshot', + arguments: { + label: 'smoke-stdio', + }, + }) + const screenshotData = requireStructuredContent(screenshot, 'desktop_screenshot') + if (!hasImageContent(screenshot)) { + throw new Error('desktop_screenshot did not return an MCP image content item') + } + if (typeof screenshotData.screenshot !== 'object' || screenshotData.screenshot == null) { + throw new Error('desktop_screenshot missing screenshot metadata') + } + + const postScreenshotCapabilities = await client.callTool({ + name: 'desktop_get_capabilities', + arguments: {}, + }) + const postScreenshotCapabilitiesData = requireStructuredContent(postScreenshotCapabilities, 'desktop_get_capabilities after screenshot') + const sessionSnapshot = (postScreenshotCapabilitiesData.session && typeof postScreenshotCapabilitiesData.session === 'object') + ? postScreenshotCapabilitiesData.session as Record + : undefined + if (!sessionSnapshot?.lastScreenshot || typeof sessionSnapshot.lastScreenshot !== 'object') { + throw new Error('desktop_get_capabilities after screenshot is missing session.lastScreenshot') + } + + const click = await client.callTool({ + name: 'desktop_click', + arguments: { + x: 100, + y: 100, + captureAfter: true, + }, + }) + const clickData = requireStructuredContent(click, 'desktop_click') + if (clickData.status !== 'approval_required') { + throw new Error(`desktop_click expected approval_required, got ${String(clickData.status)}`) + } + + const pending = await client.callTool({ + name: 'desktop_list_pending_actions', + arguments: {}, + }) + const pendingData = requireStructuredContent(pending, 'desktop_list_pending_actions') + const pendingActions = Array.isArray(pendingData.pendingActions) ? pendingData.pendingActions : [] + if (pendingActions.length === 0) { + throw new Error('desktop_list_pending_actions returned no pending action after approval_required') + } + + const pendingId = String((pendingActions[0] as Record).id || '') + if (!pendingId) { + throw new Error('first pending action missing id') + } + + const approved = await client.callTool({ + name: 'desktop_approve_pending_action', + arguments: { + id: pendingId, + }, + }) + const approvedData = requireStructuredContent(approved, 'desktop_approve_pending_action') + if (approvedData.status !== 'executed') { + throw new Error(`desktop_approve_pending_action expected executed, got ${String(approvedData.status)}`) + } + + console.info(JSON.stringify({ + ok: true, + verified: { + toolCount: tools.tools.length, + capabilities: { + hostName: (capabilitiesData.launchContext as Record).hostName, + sessionTag: (capabilitiesData.launchContext as Record).sessionTag, + coordinateSpaceBeforeScreenshot: capabilitiesData.coordinateSpace, + coordinateSpaceAfterScreenshot: postScreenshotCapabilitiesData.coordinateSpace, + }, + screenshot: screenshotData.screenshot, + approvedAction: { + id: pendingId, + status: approvedData.status, + }, + }, + }, null, 2)) + } + finally { + await client.close().catch(() => {}) + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/bin/smoke-workflow.ts b/services/computer-use-mcp/src/bin/smoke-workflow.ts new file mode 100644 index 000000000..b0f0876c0 --- /dev/null +++ b/services/computer-use-mcp/src/bin/smoke-workflow.ts @@ -0,0 +1,477 @@ +/** + * End-to-end smoke test for workflow tools. + * + * Verifies that: + * 1. `workflow_run_tests` executes all steps and returns a result. + * 2. `workflow_resume` works after an approval-paused workflow. + * 3. `desktop_get_state` reflects task progress from workflows. + * 4. All workflow tools are registered and callable. + * + * Runs against the real MCP server via stdio transport with a dry-run + * executor (no real desktop actions). Steps still flow through the full + * policy / action-executor pipeline. + * + * Usage: + * pnpm -F @proj-airi/computer-use-mcp exec tsx ./src/bin/smoke-workflow.ts + */ + +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { env, exit } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +import { appNamesMatch, findKnownAppMention } from '../app-aliases' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function requireStructuredContent(result: unknown, label: string) { + if (!result || typeof result !== 'object') + throw new Error(`${label} did not return an object result`) + + const structuredContent = (result as { structuredContent?: unknown }).structuredContent + if (!structuredContent || typeof structuredContent !== 'object') + throw new Error(`${label} missing structuredContent`) + + return structuredContent as Record +} + +function assert(condition: boolean, message: string) { + if (!condition) { + throw new Error(`Assertion failed: ${message}`) + } +} + +function createSmokeProjectDir() { + const projectPath = mkdtempSync(join(tmpdir(), 'computer-use-smoke-project-')) + writeFileSync(join(projectPath, 'README.md'), '# smoke project\n', 'utf8') + return projectPath +} + +async function createClient(overrides: Record = {}): Promise { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = (env.COMPUTER_USE_SMOKE_SERVER_ARGS || 'start').split(/\s+/).filter(Boolean) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: 'dry-run', + COMPUTER_USE_SESSION_TAG: 'smoke-workflow', + COMPUTER_USE_ALLOWED_BOUNDS: '0,0,1920,1080', + COMPUTER_USE_OPENABLE_APPS: 'Finder,Terminal,Cursor,Visual Studio Code,Google Chrome', + ...overrides, + }, + stderr: 'pipe', + }) + + const client = new Client({ + name: '@proj-airi/computer-use-mcp-smoke-workflow', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[stderr] ${text}`) + }) + + await client.connect(transport) + return client +} + +// --------------------------------------------------------------------------- +// Test 1: Workflow tools are registered +// --------------------------------------------------------------------------- + +async function testWorkflowToolsRegistered(client: Client) { + console.info('\n=== Test 1: Workflow tools are registered ===') + + const tools = await client.listTools() + const toolNames = new Set(tools.tools.map(t => t.name)) + + const requiredTools = [ + 'workflow_open_workspace', + 'workflow_validate_workspace', + 'workflow_run_tests', + 'workflow_inspect_failure', + 'workflow_browse_and_act', + 'workflow_resume', + 'desktop_get_state', + ] + + for (const name of requiredTools) { + assert(toolNames.has(name), `missing tool: ${name}`) + console.info(` ✓ ${name}`) + } + + console.info(` Total tools: ${tools.tools.length}`) + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Test 2: workflow_run_tests with autoApprove (happy path) +// --------------------------------------------------------------------------- + +async function testWorkflowRunTestsAutoApprove(client: Client) { + console.info('\n=== Test 2: workflow_run_tests with autoApprove=true ===') + const projectPath = createSmokeProjectDir() + + const result = await client.callTool({ + name: 'workflow_run_tests', + arguments: { + projectPath, + testCommand: 'echo "all tests passed"', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_run_tests') + console.info(` Status: ${data.status}`) + console.info(` Workflow: ${data.workflow}`) + + const stepResults = data.stepResults as Array<{ label: string, succeeded: boolean, explanation: string }> + for (const step of stepResults) { + const icon = step.succeeded ? '✓' : '✗' + console.info(` ${icon} ${step.label}`) + } + + // With autoApprove + dry-run, the workflow should complete. + // The dry-run executor will handle actions, and autoApprove skips the approval queue. + assert( + data.status === 'completed' || data.status === 'failed', + `expected completed or failed, got ${data.status}`, + ) + + console.info(' PASSED') + return data +} + +// --------------------------------------------------------------------------- +// Test 2b: workflow_open_workspace with autoApprove (happy path) +// --------------------------------------------------------------------------- + +async function testWorkflowOpenWorkspace(client: Client) { + console.info('\n=== Test 2b: workflow_open_workspace with autoApprove=true ===') + const projectPath = createSmokeProjectDir() + + const result = await client.callTool({ + name: 'workflow_open_workspace', + arguments: { + projectPath, + ideApp: 'VS Code', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_open_workspace') + console.info(` Status: ${data.status}`) + console.info(` Workflow: ${data.workflow}`) + + const stepResults = data.stepResults as Array<{ label: string, succeeded: boolean }> + assert(stepResults.some(step => step.label.includes('Finder')), 'expected Finder step') + assert( + stepResults.some(step => appNamesMatch(findKnownAppMention(step.label), 'Visual Studio Code')), + 'expected VS Code step', + ) + assert( + data.status === 'completed' || data.status === 'failed', + `expected completed or failed, got ${data.status}`, + ) + + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Test 2c: workflow_validate_workspace with autoApprove (happy path) +// --------------------------------------------------------------------------- + +async function testWorkflowValidateWorkspace(client: Client) { + console.info('\n=== Test 2c: workflow_validate_workspace with autoApprove=true ===') + const projectPath = createSmokeProjectDir() + + const result = await client.callTool({ + name: 'workflow_validate_workspace', + arguments: { + projectPath, + ideApp: 'VS Code', + changesCommand: 'printf " M smoke-workflow.ts\\n"', + checkCommand: 'echo "typecheck ok"', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_validate_workspace') + console.info(` Status: ${data.status}`) + console.info(` Workflow: ${data.workflow}`) + + const stepResults = data.stepResults as Array<{ label: string, succeeded: boolean }> + assert(stepResults.some(step => step.label === 'Confirm project working directory'), 'expected pwd validation step') + assert(stepResults.some(step => step.label === 'Inspect local changes'), 'expected changes inspection step') + assert(stepResults.some(step => step.label === 'Run workspace validation'), 'expected workspace validation step') + assert( + data.status === 'completed' || data.status === 'failed', + `expected completed or failed, got ${data.status}`, + ) + + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Test 3: desktop_get_state reflects workflow task +// --------------------------------------------------------------------------- + +async function testDesktopGetStateAfterWorkflow(client: Client) { + console.info('\n=== Test 3: desktop_get_state reflects workflow task ===') + + const result = await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }) + + const data = requireStructuredContent(result, 'desktop_get_state') + assert(data.status === 'ok', `expected ok status, got ${data.status}`) + + const runState = data.runState as Record + console.info(` Active app: ${runState.activeApp ?? 'unknown'}`) + console.info(` Terminal state: ${JSON.stringify(runState.terminalState)}`) + + // After a workflow, there should be task info (or it's already cleared). + if (runState.activeTask) { + const task = runState.activeTask as Record + console.info(` Task goal: ${task.goal}`) + console.info(` Task phase: ${task.phase}`) + } + else { + console.info(' No active task (workflow already finished)') + } + + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Test 4: workflow_run_tests with autoApprove=false → paused → resume +// --------------------------------------------------------------------------- + +async function testWorkflowPauseAndResume(client: Client) { + console.info('\n=== Test 4: workflow with autoApprove=false → pause → resume ===') + + const result = await client.callTool({ + name: 'workflow_run_tests', + arguments: { + projectPath: '/tmp/test-project', + testCommand: 'echo "tests"', + autoApprove: false, + }, + }) + + const data = requireStructuredContent(result, 'workflow_run_tests') + console.info(` Initial status: ${data.status}`) + + if (data.status === 'paused') { + console.info(' Workflow paused as expected (approval required)') + console.info(` Paused at step: ${data.pausedAtStep}`) + assert(data.resumeHint !== undefined, 'missing resumeHint in paused response') + + // Approve the pending action first. + const pending = await client.callTool({ + name: 'desktop_list_pending_actions', + arguments: {}, + }) + const pendingData = requireStructuredContent(pending, 'desktop_list_pending_actions') + const pendingActions = Array.isArray(pendingData.pendingActions) ? pendingData.pendingActions : [] + + if (pendingActions.length > 0) { + const pendingId = String((pendingActions[0] as Record).id || '') + console.info(` Approving pending action: ${pendingId}`) + + await client.callTool({ + name: 'desktop_approve_pending_action', + arguments: { id: pendingId }, + }) + } + + // Now resume the workflow. + console.info(' Calling workflow_resume...') + const resumeResult = await client.callTool({ + name: 'workflow_resume', + arguments: { approved: true, autoApprove: true }, + }) + const resumeData = requireStructuredContent(resumeResult, 'workflow_resume') + console.info(` Resume status: ${resumeData.status}`) + + const resumeSteps = resumeData.stepResults as Array<{ label: string, succeeded: boolean }> + for (const step of resumeSteps) { + const icon = step.succeeded ? '✓' : '✗' + console.info(` ${icon} ${step.label}`) + } + + // After resume, the workflow should be completed or at least further along. + console.info(` Final resume status: ${resumeData.status}`) + } + else if (data.status === 'completed') { + // In dry-run mode, the policy might not require approval for some actions. + console.info(' Workflow completed without needing approval (dry-run policy)') + } + else { + console.info(` Workflow ended with status: ${data.status} (may have failed steps)`) + } + + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Test 5: workflow_resume with no suspended workflow +// --------------------------------------------------------------------------- + +async function testResumeNoSuspendedWorkflow(client: Client) { + console.info('\n=== Test 5: workflow_resume with no suspended workflow ===') + + const result = await client.callTool({ + name: 'workflow_resume', + arguments: {}, + }) + + // Should be an error. + const isError = (result as { isError?: boolean }).isError + assert(isError === true, 'expected error when no workflow is suspended') + + const data = requireStructuredContent(result, 'workflow_resume') + assert(data.reason === 'no_suspended_workflow', `expected no_suspended_workflow, got ${data.reason}`) + + console.info(' Correctly returned error for no suspended workflow') + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Test 6: workflow_inspect_failure +// --------------------------------------------------------------------------- + +async function testWorkflowInspectFailure(client: Client) { + console.info('\n=== Test 6: workflow_inspect_failure ===') + + const result = await client.callTool({ + name: 'workflow_inspect_failure', + arguments: { + ideApp: 'Cursor', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_inspect_failure') + console.info(` Status: ${data.status}`) + + const stepResults = data.stepResults as Array<{ label: string, succeeded: boolean }> + for (const step of stepResults) { + const icon = step.succeeded ? '✓' : '✗' + console.info(` ${icon} ${step.label}`) + } + + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Test 7: workflow_browse_and_act +// --------------------------------------------------------------------------- + +async function testWorkflowBrowseAndAct(client: Client) { + console.info('\n=== Test 7: workflow_browse_and_act ===') + + const result = await client.callTool({ + name: 'workflow_browse_and_act', + arguments: { + app: 'Google Chrome', + goal: 'Check the homepage', + autoApprove: true, + }, + }) + + const data = requireStructuredContent(result, 'workflow_browse_and_act') + console.info(` Status: ${data.status}`) + + // Reroute contract shape check: when the strategy returns reroute_required + // the formatter must emit the stable workflow_reroute contract. + if (data.kind === 'workflow_reroute' && data.status === 'reroute_required') { + console.info(' → Reroute detected, verifying contract shape') + assert(typeof data.workflow === 'string', 'reroute must include workflow name') + const reroute = data.reroute as Record | undefined + assert(reroute != null && typeof reroute === 'object', 'reroute must include reroute detail') + assert(typeof reroute.recommendedSurface === 'string', 'reroute.recommendedSurface must be string') + assert(typeof reroute.suggestedTool === 'string', 'reroute.suggestedTool must be string') + assert(typeof reroute.strategyReason === 'string', 'reroute.strategyReason must be string') + assert(typeof reroute.explanation === 'string', 'reroute.explanation must be string') + console.info(` ✓ Reroute contract valid (recommended: ${reroute.recommendedSurface})`) + } + else { + const stepResults = data.stepResults as Array<{ label: string, succeeded: boolean }> + for (const step of stepResults) { + const icon = step.succeeded ? '✓' : '✗' + console.info(` ${icon} ${step.label}`) + } + } + + console.info(' PASSED') +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.info('╔════════════════════════════════════════════════╗') + console.info('║ Computer Use MCP — Workflow E2E Smoke Test ║') + console.info('╚════════════════════════════════════════════════╝') + + // Test with approval_mode=never (auto-approve all). + console.info('\n--- Phase 1: approval_mode=never ---') + const clientNoApproval = await createClient({ + COMPUTER_USE_APPROVAL_MODE: 'never', + }) + + try { + await testWorkflowToolsRegistered(clientNoApproval) + await testWorkflowRunTestsAutoApprove(clientNoApproval) + await testWorkflowOpenWorkspace(clientNoApproval) + await testWorkflowValidateWorkspace(clientNoApproval) + await testDesktopGetStateAfterWorkflow(clientNoApproval) + await testResumeNoSuspendedWorkflow(clientNoApproval) + await testWorkflowInspectFailure(clientNoApproval) + await testWorkflowBrowseAndAct(clientNoApproval) + } + finally { + await clientNoApproval.close().catch(() => {}) + } + + // Test with approval_mode=actions (per-step approval required). + console.info('\n--- Phase 2: approval_mode=actions (autoApprove=false for pause/resume) ---') + const clientWithApproval = await createClient({ + COMPUTER_USE_APPROVAL_MODE: 'actions', + }) + + try { + await testWorkflowPauseAndResume(clientWithApproval) + } + finally { + await clientWithApproval.close().catch(() => {}) + } + + console.info('\n╔════════════════════════════════════════════════╗') + console.info('║ ALL WORKFLOW SMOKE TESTS PASSED ║') + console.info('╚════════════════════════════════════════════════╝') +} + +main().catch((error) => { + console.error('\n❌ SMOKE TEST FAILED') + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) +}) diff --git a/services/computer-use-mcp/src/browser-dom/cdp-bridge.test.ts b/services/computer-use-mcp/src/browser-dom/cdp-bridge.test.ts new file mode 100644 index 000000000..4c4da6083 --- /dev/null +++ b/services/computer-use-mcp/src/browser-dom/cdp-bridge.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from 'vitest' + +import { CdpBridge } from '../browser-dom/cdp-bridge' + +describe('cdpBridge', () => { + it('creates with correct initial status', () => { + const bridge = new CdpBridge({ + cdpUrl: 'http://localhost:9222', + requestTimeoutMs: 10_000, + }) + + const status = bridge.getStatus() + expect(status.cdpUrl).toBe('http://localhost:9222') + expect(status.connected).toBe(false) + expect(status.pageTitle).toBeUndefined() + expect(status.pageUrl).toBeUndefined() + expect(status.lastError).toBeUndefined() + }) + + it('formats empty AX tree correctly', () => { + const bridge = new CdpBridge({ + cdpUrl: 'http://localhost:9222', + requestTimeoutMs: 10_000, + }) + + const text = bridge.formatAXTreeAsText({ + nodes: [], + pageUrl: 'https://example.com', + pageTitle: 'Example', + capturedAt: '2025-01-01T00:00:00.000Z', + }) + + expect(text).toContain('[Browser AXTree] Example (https://example.com)') + }) + + it('formats AX tree with nodes', () => { + const bridge = new CdpBridge({ + cdpUrl: 'http://localhost:9222', + requestTimeoutMs: 10_000, + }) + + const text = bridge.formatAXTreeAsText({ + nodes: [ + { + nodeId: '1', + role: 'RootWebArea', + name: 'Example Page', + children: [ + { + nodeId: '2', + role: 'heading', + name: 'Welcome', + children: [], + }, + { + nodeId: '3', + role: 'textbox', + name: 'Search', + value: 'hello world', + focused: true, + children: [], + }, + ], + }, + ], + pageUrl: 'https://example.com', + pageTitle: 'Example', + capturedAt: '2025-01-01T00:00:00.000Z', + }) + + expect(text).toContain('RootWebArea "Example Page"') + expect(text).toContain('heading "Welcome"') + expect(text).toContain('textbox "Search" val="hello world" [focused]') + }) + + it('truncates long values in AX tree format', () => { + const bridge = new CdpBridge({ + cdpUrl: 'http://localhost:9222', + requestTimeoutMs: 10_000, + }) + + const longValue = 'A'.repeat(200) + const text = bridge.formatAXTreeAsText({ + nodes: [ + { + nodeId: '1', + role: 'textbox', + value: longValue, + children: [], + }, + ], + pageUrl: 'https://example.com', + pageTitle: 'Example', + capturedAt: '2025-01-01T00:00:00.000Z', + }) + + expect(text).toContain('...') + expect(text).not.toContain('A'.repeat(200)) + }) + + it('rejects send when not connected', async () => { + const bridge = new CdpBridge({ + cdpUrl: 'http://localhost:9222', + requestTimeoutMs: 10_000, + }) + + await expect(bridge.send('Runtime.evaluate', {})).rejects.toThrow('CDP bridge is not connected') + }) + + it('close is safe to call when not connected', async () => { + const bridge = new CdpBridge({ + cdpUrl: 'http://localhost:9222', + requestTimeoutMs: 10_000, + }) + + // Should not throw + await bridge.close() + expect(bridge.getStatus().connected).toBe(false) + }) +}) diff --git a/services/computer-use-mcp/src/browser-dom/cdp-bridge.ts b/services/computer-use-mcp/src/browser-dom/cdp-bridge.ts new file mode 100644 index 000000000..651640413 --- /dev/null +++ b/services/computer-use-mcp/src/browser-dom/cdp-bridge.ts @@ -0,0 +1,390 @@ +/** + * Direct Chrome DevTools Protocol (CDP) bridge for browser DOM access. + * + * Connects to Chrome/Chromium via the CDP WebSocket endpoint (e.g. + * http://localhost:9222) to provide: + * - Accessibility tree snapshots (via Accessibility domain) + * - DOM queries and manipulation (via Runtime.evaluate) + * - Page navigation and observation + * + * This complements the extension-based bridge by not requiring a Chrome + * extension to be installed — only that Chrome is launched with + * --remote-debugging-port. + */ + +import { WebSocket } from 'ws' + +export interface CdpBridgeConfig { + /** CDP endpoint URL, e.g. http://localhost:9222 */ + cdpUrl: string + /** Request timeout in milliseconds */ + requestTimeoutMs: number +} + +export interface CdpBridgeStatus { + cdpUrl: string + connected: boolean + pageTitle?: string + pageUrl?: string + lastError?: string +} + +export interface CdpAXNode { + nodeId: string + role: string + name?: string + value?: string + description?: string + bounds?: { x: number, y: number, width: number, height: number } + focused?: boolean + children: CdpAXNode[] +} + +export interface CdpAXSnapshot { + nodes: CdpAXNode[] + pageUrl: string + pageTitle: string + capturedAt: string +} + +interface CdpMessage { + id: number + method: string + params?: Record +} + +interface CdpResponse { + id: number + result?: any + error?: { code: number, message: string } +} + +interface PendingCdpRequest { + resolve: (value: any) => void + reject: (error: Error) => void + timeoutId: NodeJS.Timeout +} + +interface CdpTargetInfo { + id: string + type: string + title: string + url: string + webSocketDebuggerUrl?: string +} + +export class CdpBridge { + private socket?: WebSocket + private nextId = 1 + private pending = new Map() + private status: CdpBridgeStatus + + constructor(private readonly config: CdpBridgeConfig) { + this.status = { + cdpUrl: config.cdpUrl, + connected: false, + } + } + + getStatus(): CdpBridgeStatus { + return { ...this.status } + } + + /** + * Connect to the first available page target via CDP. + */ + async connect(): Promise { + // Fetch available targets from the CDP HTTP endpoint + const listUrl = `${this.config.cdpUrl}/json/list` + const response = await fetch(listUrl) + + if (!response.ok) { + throw new Error(`CDP target list failed: ${response.status} ${response.statusText}`) + } + + const targets = await response.json() as CdpTargetInfo[] + const pageTarget = targets.find(t => t.type === 'page' && t.webSocketDebuggerUrl) + + if (!pageTarget?.webSocketDebuggerUrl) { + throw new Error('no page target with WebSocket debugger URL found') + } + + await this.connectToTarget(pageTarget) + } + + /** + * Connect to a specific CDP target. + */ + async connectToTarget(target: CdpTargetInfo): Promise { + if (this.socket) { + this.socket.close() + this.socket = undefined + } + + const wsUrl = target.webSocketDebuggerUrl! + + await new Promise((resolve, reject) => { + const socket = new WebSocket(wsUrl) + + socket.on('open', () => { + this.socket = socket + this.status.connected = true + this.status.pageTitle = target.title + this.status.pageUrl = target.url + this.status.lastError = undefined + resolve() + }) + + socket.on('message', (data) => { + this.handleMessage(data) + }) + + socket.on('close', () => { + this.socket = undefined + this.status.connected = false + }) + + socket.on('error', (error) => { + this.status.lastError = error instanceof Error ? error.message : String(error) + if (!this.socket) { + reject(new Error(`CDP WebSocket connection failed: ${this.status.lastError}`)) + } + }) + }) + + // Enable required CDP domains + await this.send('Accessibility.enable', {}) + await this.send('DOM.enable', {}) + await this.send('Runtime.enable', {}) + } + + /** + * Close the CDP connection. + */ + async close(): Promise { + for (const [id, pending] of this.pending.entries()) { + clearTimeout(pending.timeoutId) + pending.reject(new Error(`CDP bridge closed before completing request ${id}`)) + } + this.pending.clear() + + if (this.socket) { + this.socket.close() + this.socket = undefined + } + this.status.connected = false + } + + /** + * Send a CDP command and wait for the response. + */ + async send(method: string, params: Record = {}): Promise { + if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { + throw new Error(`CDP bridge is not connected (status: ${this.status.lastError || 'disconnected'})`) + } + + const id = this.nextId++ + const message: CdpMessage = { id, method, params } + + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.pending.delete(id) + reject(new Error(`CDP command ${method} timed out after ${this.config.requestTimeoutMs}ms`)) + }, this.config.requestTimeoutMs) + + this.pending.set(id, { resolve, reject, timeoutId }) + this.socket!.send(JSON.stringify(message)) + }) + } + + /** + * Get the full accessibility tree of the current page. + */ + async getAccessibilityTree(): Promise { + const result = await this.send('Accessibility.getFullAXTree', {}) + const nodes = (result.nodes ?? []) as any[] + + // Build a tree from the flat CDP AX node list + const nodeMap = new Map() + const rootNodes: CdpAXNode[] = [] + + for (const raw of nodes) { + const node: CdpAXNode = { + nodeId: raw.nodeId ?? '', + role: raw.role?.value ?? '', + name: raw.name?.value, + value: raw.value?.value, + description: raw.description?.value, + focused: raw.properties?.some((p: any) => p.name === 'focused' && p.value?.value === true), + children: [], + } + nodeMap.set(node.nodeId, node) + } + + // Wire parent-child relationships + for (const raw of nodes) { + const parentNode = nodeMap.get(raw.nodeId ?? '') + if (!parentNode) + continue + + const childIds = raw.childIds ?? [] + for (const childId of childIds) { + const child = nodeMap.get(childId) + if (child) { + parentNode.children.push(child) + } + } + + if (!raw.parentId && parentNode) { + rootNodes.push(parentNode) + } + } + + return { + nodes: rootNodes, + pageUrl: this.status.pageUrl ?? '', + pageTitle: this.status.pageTitle ?? '', + capturedAt: new Date().toISOString(), + } + } + + /** + * Evaluate a JavaScript expression in the page context. + */ + async evaluate(expression: string): Promise { + const result = await this.send('Runtime.evaluate', { + expression, + returnByValue: true, + awaitPromise: true, + }) + + if (result.exceptionDetails) { + const text = result.exceptionDetails.text ?? result.exceptionDetails.exception?.description ?? 'evaluation failed' + throw new Error(`CDP evaluate error: ${text}`) + } + + return result.result?.value + } + + /** + * Navigate the current page to a URL. + */ + async navigate(url: string): Promise { + await this.send('Page.navigate', { url }) + } + + /** + * Take a screenshot of the current page. + * Returns base64-encoded PNG data. + */ + async screenshot(options?: { format?: 'png' | 'jpeg', quality?: number }): Promise { + const result = await this.send('Page.captureScreenshot', { + format: options?.format ?? 'png', + quality: options?.quality, + }) + return result.data + } + + /** + * Get interactive DOM elements via Runtime.evaluate, similar to + * the extension's collectFrameDOM. Injects a small script that + * collects visible interactive elements. + */ + async collectInteractiveElements(maxElements = 200): Promise { + const expression = ` + (() => { + const selectors = 'a,button,input,select,textarea,[role="button"],[role="link"],[role="checkbox"],[role="radio"],[role="tab"],[role="menuitem"],[contenteditable="true"]'; + const elements = [...document.querySelectorAll(selectors)]; + const results = []; + for (const el of elements) { + if (results.length >= ${maxElements}) break; + const rect = el.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) continue; + const visible = rect.top < window.innerHeight && rect.bottom > 0 && rect.left < window.innerWidth && rect.right > 0; + if (!visible) continue; + results.push({ + tag: el.tagName.toLowerCase(), + id: el.id || undefined, + name: el.getAttribute('name') || undefined, + type: el.getAttribute('type') || undefined, + text: (el.innerText || '').slice(0, 120) || undefined, + value: el.value !== undefined ? String(el.value).slice(0, 120) : undefined, + href: el.href || undefined, + placeholder: el.placeholder || undefined, + disabled: el.disabled || undefined, + checked: el.checked || undefined, + role: el.getAttribute('role') || undefined, + rect: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) }, + center: { x: Math.round(rect.x + rect.width / 2), y: Math.round(rect.y + rect.height / 2) }, + }); + } + return results; + })() + ` + + return await this.evaluate(expression) + } + + /** + * Format the CDP accessibility tree as text for LLM context. + */ + formatAXTreeAsText(snapshot: CdpAXSnapshot): string { + const lines: string[] = [] + lines.push(`[Browser AXTree] ${snapshot.pageTitle} (${snapshot.pageUrl})`) + + function walk(node: CdpAXNode, depth: number) { + const prefix = ' '.repeat(depth) + const parts: string[] = [node.role || '(no role)'] + + if (node.name) { + parts.push(`"${node.name}"`) + } + if (node.value) { + const truncated = node.value.length > 80 ? `${node.value.slice(0, 77)}...` : node.value + parts.push(`val="${truncated}"`) + } + if (node.focused) { + parts.push('[focused]') + } + + lines.push(`${prefix}${parts.join(' ')}`) + + for (const child of node.children) { + walk(child, depth + 1) + } + } + + for (const root of snapshot.nodes) { + walk(root, 0) + } + + return lines.join('\n') + } + + private handleMessage(raw: any) { + let data: CdpResponse | undefined + try { + data = JSON.parse(String(raw)) as CdpResponse + } + catch { + return + } + + if (!data || typeof data.id !== 'number') + return + + const pending = this.pending.get(data.id) + if (!pending) + return + + clearTimeout(pending.timeoutId) + this.pending.delete(data.id) + + if (data.error) { + pending.reject(new Error(`CDP error: ${data.error.message} (${data.error.code})`)) + } + else { + pending.resolve(data.result) + } + } +} diff --git a/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts b/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts new file mode 100644 index 000000000..192c067cf --- /dev/null +++ b/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { WebSocket } from 'ws' + +import { BrowserDomExtensionBridge } from './extension-bridge' + +describe('browserDomExtensionBridge', () => { + let bridge: BrowserDomExtensionBridge | undefined + let client: WebSocket | undefined + + afterEach(async () => { + client?.close() + client = undefined + await bridge?.close() + bridge = undefined + }) + + it('round-trips actions over the extension websocket bridge', async () => { + bridge = new BrowserDomExtensionBridge({ + enabled: true, + host: '127.0.0.1', + port: 0, + requestTimeoutMs: 1_000, + }) + await bridge.start() + + const status = bridge.getStatus() + client = new WebSocket(`ws://${status.host}:${status.port}`) + + client.on('message', (raw) => { + const data = JSON.parse(String(raw)) as Record + if (typeof data.id !== 'string') + return + + if (data.action === 'getActiveTab') { + client!.send(JSON.stringify({ + id: data.id, + ok: true, + result: { + title: 'AIRI Demo Tab', + url: 'https://example.com/demo', + }, + })) + } + }) + + await new Promise((resolve, reject) => { + client!.once('open', () => { + client!.send(JSON.stringify({ + type: 'hello', + source: 'test-extension', + version: 'bridge-test', + })) + resolve() + }) + client!.once('error', reject) + }) + + const activeTab = await bridge.getActiveTab() + + expect(activeTab).toEqual({ + title: 'AIRI Demo Tab', + url: 'https://example.com/demo', + }) + expect(bridge.getStatus().connected).toBe(true) + expect(bridge.getStatus().lastHello?.source).toBe('test-extension') + }) +}) diff --git a/services/computer-use-mcp/src/browser-dom/extension-bridge.ts b/services/computer-use-mcp/src/browser-dom/extension-bridge.ts new file mode 100644 index 000000000..467d8f254 --- /dev/null +++ b/services/computer-use-mcp/src/browser-dom/extension-bridge.ts @@ -0,0 +1,430 @@ +import type { AddressInfo } from 'node:net' + +import type { + BrowserDomBridgeConfig, + BrowserDomBridgeHello, + BrowserDomBridgeStatus, + BrowserDomFrameResult, +} from '../types' + +import { randomUUID } from 'node:crypto' + +import { WebSocket, WebSocketServer } from 'ws' + +interface PendingBridgeRequest { + reject: (error: Error) => void + resolve: (value: unknown) => void + timeoutId: NodeJS.Timeout +} + +function asError(error: unknown, fallback: string) { + if (error instanceof Error) + return error + + return new Error(typeof error === 'string' && error.trim() ? error : fallback) +} + +function toRecord(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) + return undefined + + return value as Record +} + +function unwrapResultPayload(value: unknown): T | undefined { + const record = toRecord(value) + if (!record) + return value as T + + if ('data' in record) + return record.data as T + + return value as T +} + +export class BrowserDomExtensionBridge { + private readonly pending = new Map() + private server?: WebSocketServer + private socket?: WebSocket + private started = false + private status: BrowserDomBridgeStatus + + constructor(private readonly config: BrowserDomBridgeConfig) { + this.status = { + enabled: config.enabled, + host: config.host, + port: config.port, + connected: false, + pendingRequests: 0, + } + } + + async start() { + if (!this.config.enabled || this.started) + return + + this.started = true + + try { + const server = await new Promise((resolve, reject) => { + const nextServer = new WebSocketServer({ + host: this.config.host, + port: this.config.port, + }) + + const cleanup = () => { + nextServer.off('listening', onListening) + nextServer.off('error', onError) + } + + const onListening = () => { + cleanup() + resolve(nextServer) + } + + const onError = (error: Error) => { + cleanup() + reject(error) + } + + nextServer.once('listening', onListening) + nextServer.once('error', onError) + }) + + this.server = server + const address = server.address() + if (address && typeof address === 'object') { + this.status.host = (address as AddressInfo).address + this.status.port = (address as AddressInfo).port + } + + server.on('connection', (socket) => { + if (this.socket && this.socket !== socket) { + this.socket.close() + } + + this.socket = socket + this.status.connected = true + this.status.lastError = undefined + + socket.on('message', data => this.handleMessage(data)) + socket.on('close', () => { + if (this.socket === socket) { + this.socket = undefined + this.status.connected = false + } + }) + socket.on('error', (error) => { + this.status.lastError = asError(error, 'browser dom bridge socket error').message + }) + }) + + server.on('error', (error) => { + this.status.lastError = asError(error, 'browser dom bridge server error').message + }) + } + catch (error) { + this.status.lastError = asError(error, 'failed to start browser dom bridge').message + } + } + + async close() { + for (const [requestId, pending] of this.pending.entries()) { + clearTimeout(pending.timeoutId) + pending.reject(new Error(`browser dom bridge closed before completing request ${requestId}`)) + } + this.pending.clear() + this.status.pendingRequests = 0 + + if (this.socket) { + this.socket.close() + this.socket = undefined + } + this.status.connected = false + + if (this.server) { + const server = this.server + this.server = undefined + await new Promise((resolve) => { + server.close(() => resolve()) + }) + } + } + + getStatus(): BrowserDomBridgeStatus { + return { + ...this.status, + lastHello: this.status.lastHello ? { ...this.status.lastHello } : undefined, + } + } + + async callAction(action: string, payload: Record = {}): Promise { + if (!this.config.enabled) { + throw new Error('browser dom bridge is disabled') + } + + if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { + throw new Error(this.status.lastError || 'browser dom bridge is not connected') + } + + const id = randomUUID() + const requestPayload = { + id, + action, + ...payload, + } + + const result = await new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.pending.delete(id) + this.status.pendingRequests = this.pending.size + reject(new Error(`browser dom bridge timed out waiting for ${action}`)) + }, this.config.requestTimeoutMs) + + this.pending.set(id, { + resolve: value => resolve(value as TResult), + reject, + timeoutId, + }) + this.status.pendingRequests = this.pending.size + + this.socket!.send(JSON.stringify(requestPayload), (error) => { + if (!error) + return + + const pending = this.pending.get(id) + if (!pending) + return + + clearTimeout(pending.timeoutId) + this.pending.delete(id) + this.status.pendingRequests = this.pending.size + pending.reject(asError(error, `failed to send ${action} to browser dom bridge`)) + }) + }) + + return result + } + + async getActiveTab() { + return await this.callAction | null>('getActiveTab') + } + + async getAllFrames(params: { tabId?: number } = {}) { + return await this.callAction>>('getAllFrames', params) + } + + async readAllFramesDom(params: { + tabId?: number + frameIds?: number[] + includeText?: boolean + maxElements?: number + } = {}) { + return await this.callAction>>>('readAllFramesDOM', { + tabId: params.tabId, + frameIds: params.frameIds, + opts: { + includeText: params.includeText ?? true, + maxElements: params.maxElements ?? 200, + }, + }) + } + + async findElements(params: { + selector: string + maxResults?: number + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('findElements', { + selector: params.selector, + max: params.maxResults ?? 10, + tabId: params.tabId, + frameIds: params.frameIds, + }) + } + + async readInputValue(params: { + selector: string + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('readInputValue', params) + } + + async setInputValue(params: { + selector: string + value: string + simulateKeystrokes?: boolean + blur?: boolean + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('setInputValue', { + selector: params.selector, + value: params.value, + opts: { + simulateKeystrokes: params.simulateKeystrokes ?? false, + blur: params.blur ?? true, + }, + tabId: params.tabId, + frameIds: params.frameIds, + }) + } + + async checkCheckbox(params: { + selector: string + checked?: boolean + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('checkCheckbox', params) + } + + async selectOption(params: { + selector: string + value: string + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('selectOption', params) + } + + async getElementAttributes(params: { + selector: string + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('getElementAttributes', params) + } + + async getComputedStyles(params: { + selector: string + properties?: string[] + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('getComputedStyles', { + selector: params.selector, + properties: params.properties, + tabId: params.tabId, + frameIds: params.frameIds, + }) + } + + async triggerEvent(params: { + selector: string + eventName: string + eventType?: string + opts?: Record + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('triggerEvent', { + selector: params.selector, + eventName: params.eventName, + opts: { + ...params.opts, + ...(params.eventType ? { type: params.eventType } : {}), + }, + tabId: params.tabId, + frameIds: params.frameIds, + }) + } + + async waitForElement(params: { + selector: string + timeoutMs?: number + tabId?: number + frameIds?: number[] + }) { + return await this.callAction>>>('waitForElement', { + selector: params.selector, + timeoutMs: params.timeoutMs ?? this.config.requestTimeoutMs, + tabId: params.tabId, + frameIds: params.frameIds, + }) + } + + async clickSelector(params: { + selector: string + tabId?: number + frameIds?: number[] + }) { + const targets = await this.callAction>>>('getClickTarget', { + selector: params.selector, + tabId: params.tabId, + frameIds: params.frameIds, + }) + + const target = targets.find((entry) => { + const payload = unwrapResultPayload<{ x?: number, y?: number }>(entry.result) + return typeof payload?.x === 'number' && typeof payload?.y === 'number' + }) + + if (!target) { + throw new Error(`browser dom bridge could not find a clickable target for selector "${params.selector}"`) + } + + const payload = unwrapResultPayload<{ x: number, y: number, element?: Record }>(target.result) + const clickResults = await this.callAction>>>('clickAt', { + x: payload!.x, + y: payload!.y, + tabId: params.tabId, + frameIds: [target.frameId], + }) + + return { + targetFrameId: target.frameId, + targetPoint: { + x: payload!.x, + y: payload!.y, + }, + targetElement: payload?.element, + clickResults, + } + } + + private handleMessage(raw: WebSocket.RawData) { + let data: Record | undefined + try { + data = JSON.parse(String(raw)) as Record + } + catch { + return + } + + if (data.type === 'hello') { + const nextHello: BrowserDomBridgeHello = { + source: typeof data.source === 'string' ? data.source : undefined, + version: typeof data.version === 'string' ? data.version : undefined, + connectedAt: new Date().toISOString(), + } + this.status.lastHello = nextHello + this.status.lastError = undefined + return + } + + const requestId = typeof data.id === 'string' ? data.id : undefined + if (!requestId) + return + + const pending = this.pending.get(requestId) + if (!pending) + return + + clearTimeout(pending.timeoutId) + this.pending.delete(requestId) + this.status.pendingRequests = this.pending.size + + if (data.ok === false) { + const message = typeof data.error === 'string' && data.error.trim() + ? data.error + : 'browser dom bridge request failed' + pending.reject(new Error(message)) + return + } + + pending.resolve(data.result) + } +} diff --git a/services/computer-use-mcp/src/config.test.ts b/services/computer-use-mcp/src/config.test.ts new file mode 100644 index 000000000..e4f68bb8c --- /dev/null +++ b/services/computer-use-mcp/src/config.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, it } from 'vitest' + +import { resolveComputerUseConfig } from './config' + +const originalEnv = { ...process.env } + +afterEach(() => { + for (const key of Object.keys(process.env)) { + if (!(key in originalEnv)) { + delete process.env[key] + } + } + + for (const [key, value] of Object.entries(originalEnv)) { + process.env[key] = value + } +}) + +describe('resolveComputerUseConfig', () => { + it('parses remote linux-x11 settings from env', () => { + process.env.COMPUTER_USE_EXECUTOR = 'linux-x11' + process.env.COMPUTER_USE_REMOTE_SSH_HOST = '20.196.212.37' + process.env.COMPUTER_USE_REMOTE_SSH_USER = 'airi' + process.env.COMPUTER_USE_REMOTE_SSH_PORT = '2201' + process.env.COMPUTER_USE_REMOTE_RUNNER_COMMAND = '~/.local/bin/custom-runner' + process.env.COMPUTER_USE_REMOTE_DISPLAY_SIZE = '1366x768' + process.env.COMPUTER_USE_REMOTE_OBSERVATION_BASE_URL = 'http://20.196.212.37:8765/observations' + process.env.COMPUTER_USE_REMOTE_OBSERVATION_TOKEN = 'observation-token' + process.env.COMPUTER_USE_ALLOWED_BOUNDS = '0,0,1366,768' + + const config = resolveComputerUseConfig() + + expect(config.executor).toBe('linux-x11') + expect(config.remoteSshHost).toBe('20.196.212.37') + expect(config.remoteSshUser).toBe('airi') + expect(config.remoteSshPort).toBe(2201) + expect(config.remoteRunnerCommand).toBe('~/.local/bin/custom-runner') + expect(config.remoteDisplaySize).toEqual({ + width: 1366, + height: 768, + }) + expect(config.remoteObservationBaseUrl).toBe('http://20.196.212.37:8765/observations') + expect(config.remoteObservationServePort).toBe(8765) + expect(config.remoteObservationToken).toBe('observation-token') + expect(config.allowedBounds).toEqual({ + x: 0, + y: 0, + width: 1366, + height: 768, + }) + expect(config.permissionChainHint).toContain('ssh -> remote desktop-runner') + expect(config.requireAllowedBoundsForMutatingActions).toBe(true) + }) + + it('defaults macos-local to deny-only window policy and local shell runner settings', () => { + process.env.COMPUTER_USE_EXECUTOR = 'macos-local' + + const config = resolveComputerUseConfig() + + expect(config.executor).toBe('macos-local') + expect(config.openableApps).toEqual(['Finder', 'Terminal', 'Cursor', 'Visual Studio Code', 'Google Chrome']) + expect(config.denyApps).toContain('airi') + expect(config.terminalShell).toBeTruthy() + expect(config.permissionChainHint).toContain('swift/quartz + open') + expect(config.requireAllowedBoundsForMutatingActions).toBe(false) + expect(config.requireCoordinateAlignmentForMutatingActions).toBe(false) + expect(config.binaries.pbcopy).toBe('pbcopy') + expect(config.binaries.pbpaste).toBe('pbpaste') + }) + + it('enables the browser dom bridge by default and respects overrides', () => { + process.env.COMPUTER_USE_BROWSER_DOM_BRIDGE_PORT = '8876' + process.env.COMPUTER_USE_BROWSER_DOM_BRIDGE_TIMEOUT_MS = '4500' + + const config = resolveComputerUseConfig() + + expect(config.browserDomBridge.enabled).toBe(true) + expect(config.browserDomBridge.host).toBe('127.0.0.1') + expect(config.browserDomBridge.port).toBe(8876) + expect(config.browserDomBridge.requestTimeoutMs).toBe(4500) + }) +}) diff --git a/services/computer-use-mcp/src/config.ts b/services/computer-use-mcp/src/config.ts new file mode 100644 index 000000000..505e14ee1 --- /dev/null +++ b/services/computer-use-mcp/src/config.ts @@ -0,0 +1,203 @@ +import type { ApprovalMode, Bounds, ComputerUseConfig, DisplaySize, ExecutorKind } from './types' + +import { join } from 'node:path' +import { cwd, env } from 'node:process' + +const defaultDeniedApps = [ + '1password', + 'keychain', + 'system settings', + 'activity monitor', + 'airi', +] + +const defaultOpenableApps = [ + 'Finder', + 'Terminal', + 'Cursor', + 'Visual Studio Code', + 'Google Chrome', +] + +function parseBoolean(value: string | undefined, fallback: boolean) { + if (value == null) + return fallback + + const normalized = value.trim().toLowerCase() + if (['1', 'true', 'yes', 'on'].includes(normalized)) + return true + if (['0', 'false', 'no', 'off'].includes(normalized)) + return false + return fallback +} + +function parseInteger(value: string | undefined, fallback: number) { + if (!value) + return fallback + + const parsed = Number.parseInt(value, 10) + return Number.isFinite(parsed) ? parsed : fallback +} + +function parseList(value: string | undefined, fallback: string[] = []) { + if (!value) + return fallback + + return value + .split(',') + .map(item => item.trim()) + .filter(Boolean) +} + +function parseBounds(value: string | undefined): Bounds | undefined { + if (!value) + return undefined + + const parts = value.split(',').map(item => Number.parseFloat(item.trim())) + if (parts.length !== 4 || parts.some(item => !Number.isFinite(item))) { + throw new Error(`invalid COMPUTER_USE_ALLOWED_BOUNDS: ${value}`) + } + + const [x, y, width, height] = parts + if (width <= 0 || height <= 0) { + throw new Error(`invalid COMPUTER_USE_ALLOWED_BOUNDS dimensions: ${value}`) + } + + return { x, y, width, height } +} + +function parseExecutor(value: string | undefined): ExecutorKind { + if (value === 'linux-x11' || value === 'macos-local') + return value + return 'dry-run' +} + +function parseApprovalMode(value: string | undefined): ApprovalMode { + if (value === 'never' || value === 'all' || value === 'actions') + return value + return 'actions' +} + +function parseDisplaySize(value: string | undefined, fallback: DisplaySize): DisplaySize { + if (!value) + return fallback + + const match = value.trim().match(/^(\d+)x(\d+)$/i) + if (!match) { + throw new Error(`invalid COMPUTER_USE_REMOTE_DISPLAY_SIZE: ${value}`) + } + + const width = Number.parseInt(match[1], 10) + const height = Number.parseInt(match[2], 10) + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + throw new Error(`invalid COMPUTER_USE_REMOTE_DISPLAY_SIZE dimensions: ${value}`) + } + + return { width, height } +} + +function inferPortFromUrl(value: string | undefined) { + if (!value) + return undefined + + try { + const url = new URL(value) + if (url.port) + return Number.parseInt(url.port, 10) + + return url.protocol === 'https:' ? 443 : 80 + } + catch { + return undefined + } +} + +export function resolveComputerUseConfig(): ComputerUseConfig { + const executor = parseExecutor(env.COMPUTER_USE_EXECUTOR) + const sessionRoot = env.COMPUTER_USE_SESSION_ROOT?.trim() || join(cwd(), '.computer-use-mcp') + const launchHostProcess = env.COMPUTER_USE_LAUNCH_HOST_PROCESS?.trim() + || env.TERM_PROGRAM?.trim() + || env.SHELL?.split('/').at(-1) + || 'node' + + const remoteSshHost = env.COMPUTER_USE_REMOTE_SSH_HOST?.trim() || undefined + const remoteSshUser = env.COMPUTER_USE_REMOTE_SSH_USER?.trim() || undefined + const remoteDisplaySize = parseDisplaySize(env.COMPUTER_USE_REMOTE_DISPLAY_SIZE, { + width: 1280, + height: 720, + }) + const remoteObservationBaseUrl = env.COMPUTER_USE_REMOTE_OBSERVATION_BASE_URL?.trim() || undefined + const inferredObservationPort = inferPortFromUrl(remoteObservationBaseUrl) + const remoteObservationServePort = env.COMPUTER_USE_REMOTE_OBSERVATION_SERVE_PORT?.trim() + ? parseInteger(env.COMPUTER_USE_REMOTE_OBSERVATION_SERVE_PORT, 8765) + : remoteObservationBaseUrl + ? inferredObservationPort || 8765 + : undefined + + const requireSessionTagForMutatingActions = parseBoolean( + env.COMPUTER_USE_REQUIRE_SESSION_TAG_FOR_MUTATIONS, + executor === 'linux-x11', + ) + const requireAllowedBoundsForMutatingActions = parseBoolean( + env.COMPUTER_USE_REQUIRE_ALLOWED_BOUNDS_FOR_MUTATIONS, + executor === 'linux-x11', + ) + const requireCoordinateAlignmentForMutatingActions = parseBoolean( + env.COMPUTER_USE_REQUIRE_COORDINATE_ALIGNMENT_FOR_MUTATIONS, + executor === 'linux-x11', + ) + + return { + sessionRoot, + screenshotsDir: join(sessionRoot, 'screenshots'), + auditLogPath: join(sessionRoot, 'audit.jsonl'), + executor, + approvalMode: parseApprovalMode(env.COMPUTER_USE_APPROVAL_MODE), + defaultCaptureAfter: parseBoolean(env.COMPUTER_USE_DEFAULT_CAPTURE_AFTER, true), + maxOperations: parseInteger(env.COMPUTER_USE_MAX_OPERATIONS, 80), + maxOperationUnits: parseInteger(env.COMPUTER_USE_MAX_OPERATION_UNITS, 160), + maxPendingActions: parseInteger(env.COMPUTER_USE_MAX_PENDING_ACTIONS, 24), + allowedBounds: parseBounds(env.COMPUTER_USE_ALLOWED_BOUNDS), + allowApps: parseList(env.COMPUTER_USE_ALLOW_APPS), + denyApps: parseList(env.COMPUTER_USE_DENY_APPS, defaultDeniedApps), + denyWindowTitles: parseList(env.COMPUTER_USE_DENY_WINDOW_TITLES), + openableApps: parseList(env.COMPUTER_USE_OPENABLE_APPS, defaultOpenableApps), + timeoutMs: parseInteger(env.COMPUTER_USE_TIMEOUT_MS, 15_000), + sessionTag: env.COMPUTER_USE_SESSION_TAG?.trim() || undefined, + launchHostProcess, + permissionChainHint: env.COMPUTER_USE_PERMISSION_CHAIN_HINT?.trim() + || (executor === 'linux-x11' + ? `${launchHostProcess} -> ssh -> remote desktop-runner` + : executor === 'macos-local' + ? `${launchHostProcess} -> swift/quartz + open` + : `${launchHostProcess} -> local dry-run`), + requireSessionTagForMutatingActions, + requireAllowedBoundsForMutatingActions, + requireCoordinateAlignmentForMutatingActions, + terminalShell: env.COMPUTER_USE_TERMINAL_SHELL?.trim() || env.SHELL?.trim() || '/bin/zsh', + remoteSshHost, + remoteSshUser, + remoteSshPort: parseInteger(env.COMPUTER_USE_REMOTE_SSH_PORT, 22), + remoteRunnerCommand: env.COMPUTER_USE_REMOTE_RUNNER_COMMAND?.trim() || '~/.local/bin/airi-desktop-runner', + remoteDisplaySize, + remoteObservationBaseUrl, + remoteObservationServePort, + remoteObservationToken: env.COMPUTER_USE_REMOTE_OBSERVATION_TOKEN?.trim() || undefined, + browserDomBridge: { + enabled: parseBoolean(env.COMPUTER_USE_BROWSER_DOM_BRIDGE_ENABLED, true), + host: env.COMPUTER_USE_BROWSER_DOM_BRIDGE_HOST?.trim() || '127.0.0.1', + port: parseInteger(env.COMPUTER_USE_BROWSER_DOM_BRIDGE_PORT, 8765), + requestTimeoutMs: parseInteger(env.COMPUTER_USE_BROWSER_DOM_BRIDGE_TIMEOUT_MS, 10_000), + }, + binaries: { + swift: env.COMPUTER_USE_SWIFT_BINARY?.trim() || 'swift', + osascript: env.COMPUTER_USE_OSASCRIPT_BINARY?.trim() || 'osascript', + screencapture: env.COMPUTER_USE_SCREENSHOT_BINARY?.trim() || 'screencapture', + pbcopy: env.COMPUTER_USE_PBCOPY_BINARY?.trim() || 'pbcopy', + pbpaste: env.COMPUTER_USE_PBPASTE_BINARY?.trim() || 'pbpaste', + ssh: env.COMPUTER_USE_SSH_BINARY?.trim() || 'ssh', + tar: env.COMPUTER_USE_TAR_BINARY?.trim() || 'tar', + open: env.COMPUTER_USE_OPEN_BINARY?.trim() || 'open', + }, + } +} diff --git a/services/computer-use-mcp/src/display/display.test.ts b/services/computer-use-mcp/src/display/display.test.ts new file mode 100644 index 000000000..b2087a9b9 --- /dev/null +++ b/services/computer-use-mcp/src/display/display.test.ts @@ -0,0 +1,118 @@ +import type { DisplayDescriptor, MultiDisplaySnapshot } from '../display/types' + +import { describe, expect, it } from 'vitest' + +import { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from '../display/types' + +function createTestSnapshot(): MultiDisplaySnapshot { + const mainDisplay: DisplayDescriptor = { + displayId: 1, + isMain: true, + isBuiltIn: true, + bounds: { x: 0, y: 0, width: 1440, height: 900 }, + visibleBounds: { x: 0, y: 25, width: 1440, height: 875 }, + scaleFactor: 2, + pixelWidth: 2880, + pixelHeight: 1800, + } + + const externalDisplay: DisplayDescriptor = { + displayId: 2, + isMain: false, + isBuiltIn: false, + bounds: { x: 1440, y: 0, width: 2560, height: 1440 }, + visibleBounds: { x: 1440, y: 25, width: 2560, height: 1415 }, + scaleFactor: 1, + pixelWidth: 2560, + pixelHeight: 1440, + } + + return { + displays: [mainDisplay, externalDisplay], + combinedBounds: { x: 0, y: 0, width: 4000, height: 1440 }, + capturedAt: '2025-01-01T00:00:00.000Z', + } +} + +describe('findDisplayForPoint', () => { + it('finds the main display for a point on it', () => { + const snapshot = createTestSnapshot() + const display = findDisplayForPoint(snapshot, 720, 450) + + expect(display).toBeDefined() + expect(display!.displayId).toBe(1) + expect(display!.isMain).toBe(true) + }) + + it('finds the external display for a point on it', () => { + const snapshot = createTestSnapshot() + const display = findDisplayForPoint(snapshot, 2000, 500) + + expect(display).toBeDefined() + expect(display!.displayId).toBe(2) + expect(display!.isMain).toBe(false) + }) + + it('returns undefined for a point outside all displays', () => { + const snapshot = createTestSnapshot() + const display = findDisplayForPoint(snapshot, 5000, 500) + + expect(display).toBeUndefined() + }) + + it('handles edge case at display boundary', () => { + const snapshot = createTestSnapshot() + // x=1440 is the first pixel of the external display + const display = findDisplayForPoint(snapshot, 1440, 0) + + expect(display).toBeDefined() + expect(display!.displayId).toBe(2) + }) +}) + +describe('toDisplayLocalCoord', () => { + it('converts global to local coordinates on main display', () => { + const snapshot = createTestSnapshot() + const local = toDisplayLocalCoord(snapshot.displays[0], 720, 450) + + expect(local.x).toBe(720) + expect(local.y).toBe(450) + }) + + it('converts global to local coordinates on external display', () => { + const snapshot = createTestSnapshot() + const local = toDisplayLocalCoord(snapshot.displays[1], 2000, 500) + + expect(local.x).toBe(560) // 2000 - 1440 + expect(local.y).toBe(500) + }) +}) + +describe('toGlobalCoord', () => { + it('converts local back to global on main display', () => { + const snapshot = createTestSnapshot() + const global = toGlobalCoord(snapshot.displays[0], 720, 450) + + expect(global.x).toBe(720) + expect(global.y).toBe(450) + }) + + it('converts local back to global on external display', () => { + const snapshot = createTestSnapshot() + const global = toGlobalCoord(snapshot.displays[1], 560, 500) + + expect(global.x).toBe(2000) + expect(global.y).toBe(500) + }) + + it('is inverse of toDisplayLocalCoord', () => { + const snapshot = createTestSnapshot() + const display = snapshot.displays[1] + const globalPoint = { x: 2500, y: 800 } + const local = toDisplayLocalCoord(display, globalPoint.x, globalPoint.y) + const roundTrip = toGlobalCoord(display, local.x, local.y) + + expect(roundTrip.x).toBe(globalPoint.x) + expect(roundTrip.y).toBe(globalPoint.y) + }) +}) diff --git a/services/computer-use-mcp/src/display/enumerate.ts b/services/computer-use-mcp/src/display/enumerate.ts new file mode 100644 index 000000000..b6f0b1858 --- /dev/null +++ b/services/computer-use-mcp/src/display/enumerate.ts @@ -0,0 +1,204 @@ +/** + * Multi-display enumeration and orchestration for macOS. + * + * Uses NSScreen via Swift to enumerate all connected displays, + * their geometry, scale factors, and calculate the combined bounding rect. + */ + +import type { ComputerUseConfig } from '../types' +import type { DisplayDescriptor, MultiDisplaySnapshot } from './types' + +import { platform } from 'node:process' + +import { runSwiftScript } from '../utils/swift' + +/** + * Swift source that enumerates all connected displays via NSScreen + * and CGDisplay APIs, returning their geometry and properties. + */ +function enumerateDisplaysScript(): string { + return String.raw` +import AppKit +import CoreGraphics +import Foundation + +struct DisplayJSON: Encodable { + let displayId: UInt32 + let isMain: Bool + let isBuiltIn: Bool + let bounds: BoundsJSON + let visibleBounds: BoundsJSON + let scaleFactor: Double + let pixelWidth: Int + let pixelHeight: Int +} + +struct BoundsJSON: Encodable { + let x: Int + let y: Int + let width: Int + let height: Int +} + +func toBounds(_ rect: NSRect) -> BoundsJSON { + return BoundsJSON( + x: Int(rect.origin.x.rounded()), + y: Int(rect.origin.y.rounded()), + width: Int(rect.size.width.rounded()), + height: Int(rect.size.height.rounded()) + ) +} + +var displays: [DisplayJSON] = [] + +for screen in NSScreen.screens { + let deviceDesc = screen.deviceDescription + let displayId = (deviceDesc[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value ?? 0 + + // NSScreen coordinates have origin at bottom-left. Convert to top-left + // by using the main screen's height as reference. + let mainHeight = NSScreen.screens.first?.frame.height ?? screen.frame.height + + let frame = screen.frame + let visibleFrame = screen.visibleFrame + let scale = screen.backingScaleFactor + + // Convert from bottom-left origin to top-left origin + let topLeftY = mainHeight - frame.origin.y - frame.height + let visibleTopLeftY = mainHeight - visibleFrame.origin.y - visibleFrame.height + + let bounds = BoundsJSON( + x: Int(frame.origin.x.rounded()), + y: Int(topLeftY.rounded()), + width: Int(frame.width.rounded()), + height: Int(frame.height.rounded()) + ) + + let visibleBounds = BoundsJSON( + x: Int(visibleFrame.origin.x.rounded()), + y: Int(visibleTopLeftY.rounded()), + width: Int(visibleFrame.width.rounded()), + height: Int(visibleFrame.height.rounded()) + ) + + let isMain = CGDisplayIsMain(displayId) != 0 + let isBuiltIn = CGDisplayIsBuiltin(displayId) != 0 + let pixelWidth = Int(frame.width * scale) + let pixelHeight = Int(frame.height * scale) + + displays.append(DisplayJSON( + displayId: displayId, + isMain: isMain, + isBuiltIn: isBuiltIn, + bounds: bounds, + visibleBounds: visibleBounds, + scaleFactor: scale, + pixelWidth: pixelWidth, + pixelHeight: pixelHeight + )) +} + +let encoder = JSONEncoder() +let data = try encoder.encode(displays) +print(String(data: data, encoding: .utf8)!) +` +} + +interface RawDisplay { + displayId: number + isMain: boolean + isBuiltIn: boolean + bounds: { x: number, y: number, width: number, height: number } + visibleBounds: { x: number, y: number, width: number, height: number } + scaleFactor: number + pixelWidth: number + pixelHeight: number +} + +function computeCombinedBounds(displays: DisplayDescriptor[]) { + if (displays.length === 0) { + return { x: 0, y: 0, width: 0, height: 0 } + } + + let minX = Infinity + let minY = Infinity + let maxX = -Infinity + let maxY = -Infinity + + for (const d of displays) { + minX = Math.min(minX, d.bounds.x) + minY = Math.min(minY, d.bounds.y) + maxX = Math.max(maxX, d.bounds.x + d.bounds.width) + maxY = Math.max(maxY, d.bounds.y + d.bounds.height) + } + + return { + x: minX, + y: minY, + width: maxX - minX, + height: maxY - minY, + } +} + +/** + * Enumerate all connected displays on macOS. + */ +export async function enumerateDisplays(config: ComputerUseConfig): Promise { + if (platform !== 'darwin') { + throw new Error('multi-display enumeration is only supported on macOS') + } + + const { stdout } = await runSwiftScript({ + swiftBinary: config.binaries.swift, + timeoutMs: config.timeoutMs, + source: enumerateDisplaysScript(), + }) + + const raw = JSON.parse(stdout.trim()) as RawDisplay[] + const displays: DisplayDescriptor[] = raw.map(d => ({ + displayId: d.displayId, + isMain: d.isMain, + isBuiltIn: d.isBuiltIn, + bounds: d.bounds, + visibleBounds: d.visibleBounds, + scaleFactor: d.scaleFactor, + pixelWidth: d.pixelWidth, + pixelHeight: d.pixelHeight, + })) + + return { + displays, + combinedBounds: computeCombinedBounds(displays), + capturedAt: new Date().toISOString(), + } +} + +/** + * Format the multi-display snapshot as a human/LLM-readable summary. + */ +export function formatDisplaySummary(snapshot: MultiDisplaySnapshot): string { + const lines: string[] = [] + lines.push(`[Displays] ${snapshot.displays.length} connected`) + + for (const d of snapshot.displays) { + const flags: string[] = [] + if (d.isMain) + flags.push('main') + if (d.isBuiltIn) + flags.push('built-in') + if (d.scaleFactor > 1) + flags.push(`${d.scaleFactor}x`) + + const b = d.bounds + lines.push( + ` #${d.displayId}${flags.length ? ` (${flags.join(', ')})` : ''}: ` + + `${b.width}x${b.height} @ (${b.x},${b.y}), ` + + `pixels ${d.pixelWidth}x${d.pixelHeight}`, + ) + } + + const cb = snapshot.combinedBounds + lines.push(` Combined: ${cb.width}x${cb.height} @ (${cb.x},${cb.y})`) + + return lines.join('\n') +} diff --git a/services/computer-use-mcp/src/display/index.ts b/services/computer-use-mcp/src/display/index.ts new file mode 100644 index 000000000..3334bab27 --- /dev/null +++ b/services/computer-use-mcp/src/display/index.ts @@ -0,0 +1,3 @@ +export { enumerateDisplays, formatDisplaySummary } from './enumerate' +export { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from './types' +export type { DisplayDescriptor, MultiDisplaySnapshot } from './types' diff --git a/services/computer-use-mcp/src/display/types.ts b/services/computer-use-mcp/src/display/types.ts new file mode 100644 index 000000000..de3cb3cd0 --- /dev/null +++ b/services/computer-use-mcp/src/display/types.ts @@ -0,0 +1,85 @@ +/** + * Multi-display types for macOS screen enumeration and coordinate mapping. + */ + +export interface DisplayDescriptor { + /** Display id from CGDirectDisplayID */ + displayId: number + /** Whether this is the main display */ + isMain: boolean + /** Whether the display is built-in (laptop screen) */ + isBuiltIn: boolean + /** Logical bounds in global screen coordinates */ + bounds: { + x: number + y: number + width: number + height: number + } + /** Usable area excluding menu bar / dock */ + visibleBounds: { + x: number + y: number + width: number + height: number + } + /** Backing scale factor (2.0 = Retina) */ + scaleFactor: number + /** Physical pixel dimensions */ + pixelWidth: number + pixelHeight: number +} + +export interface MultiDisplaySnapshot { + displays: DisplayDescriptor[] + /** Total bounding rect across all displays in logical coords */ + combinedBounds: { + x: number + y: number + width: number + height: number + } + capturedAt: string +} + +/** + * Given a logical screen point, find which display it belongs to. + */ +export function findDisplayForPoint( + snapshot: MultiDisplaySnapshot, + x: number, + y: number, +): DisplayDescriptor | undefined { + return snapshot.displays.find((d) => { + const b = d.bounds + return x >= b.x && x < b.x + b.width && y >= b.y && y < b.y + b.height + }) +} + +/** + * Convert a logical coordinate to the local coordinate space of a specific display. + */ +export function toDisplayLocalCoord( + display: DisplayDescriptor, + x: number, + y: number, +): { x: number, y: number } { + return { + x: x - display.bounds.x, + y: y - display.bounds.y, + } +} + +/** + * Convert display-local coordinates back to global logical coordinates. + */ +export function toGlobalCoord( + display: DisplayDescriptor, + localX: number, + localY: number, +): { x: number, y: number } { + return { + x: localX + display.bounds.x, + y: localY + display.bounds.y, + } +} diff --git a/services/computer-use-mcp/src/e2e/chat-turn.test.ts b/services/computer-use-mcp/src/e2e/chat-turn.test.ts new file mode 100644 index 000000000..4d6ad69b8 --- /dev/null +++ b/services/computer-use-mcp/src/e2e/chat-turn.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' + +import { hasCompletedChatTurn } from './chat-turn' + +describe('hasCompletedChatTurn', () => { + it('treats tool-only turns as completed even without assistant text', () => { + expect(hasCompletedChatTurn({ + chat: { + lastTurnComplete: { + at: '2026-03-10T12:50:39.178Z', + outputText: '', + toolCallCount: 13, + toolResultCount: 13, + }, + }, + })).toBe(true) + }) + + it('treats text output turns as completed', () => { + expect(hasCompletedChatTurn({ + chat: { + lastTurnComplete: { + outputText: 'hello from AIRI', + toolCallCount: 0, + toolResultCount: 0, + }, + }, + })).toBe(true) + }) + + it('ignores snapshots without a completed turn summary', () => { + expect(hasCompletedChatTurn({ + chat: { + lastTurnComplete: null, + }, + })).toBe(false) + + expect(hasCompletedChatTurn({ + chat: { + lastTurnComplete: { + outputText: '', + toolCallCount: 0, + toolResultCount: 0, + }, + }, + })).toBe(false) + }) +}) diff --git a/services/computer-use-mcp/src/e2e/chat-turn.ts b/services/computer-use-mcp/src/e2e/chat-turn.ts new file mode 100644 index 000000000..3c923d4fe --- /dev/null +++ b/services/computer-use-mcp/src/e2e/chat-turn.ts @@ -0,0 +1,36 @@ +interface ChatTurnSummaryLike { + at?: unknown + outputText?: unknown + toolCallCount?: unknown + toolResultCount?: unknown +} + +interface ChatTurnSnapshotLike { + chat?: { + lastTurnComplete?: ChatTurnSummaryLike | null + } +} + +// NOTICE: AIRI can legitimately finish a turn with tool calls/results but no +// natural-language assistant text. Treat that as a completed turn so E2E +// harnesses do not hang forever waiting for output that will never arrive. +export function hasCompletedChatTurn(snapshot: ChatTurnSnapshotLike) { + const lastTurnComplete = snapshot.chat?.lastTurnComplete + if (!lastTurnComplete || typeof lastTurnComplete !== 'object') { + return false + } + + const outputText = typeof lastTurnComplete.outputText === 'string' + ? lastTurnComplete.outputText.trim() + : '' + const toolCallCount = Number(lastTurnComplete.toolCallCount || 0) + const toolResultCount = Number(lastTurnComplete.toolResultCount || 0) + const completedAt = typeof lastTurnComplete.at === 'string' + ? lastTurnComplete.at.trim() + : '' + + return outputText.length > 0 + || toolCallCount > 0 + || toolResultCount > 0 + || completedAt.length > 0 +} diff --git a/services/computer-use-mcp/src/e2e/debug-targets.test.ts b/services/computer-use-mcp/src/e2e/debug-targets.test.ts new file mode 100644 index 000000000..7c12a36e7 --- /dev/null +++ b/services/computer-use-mcp/src/e2e/debug-targets.test.ts @@ -0,0 +1,79 @@ +import type { DebugTargetLike } from './debug-targets' + +import { describe, expect, it } from 'vitest' + +import { + + isChatSurfaceTarget, + isInspectableAiriRendererTarget, + prioritizeInspectableAiriTargets, +} from './debug-targets' + +function createTarget(partial: Partial): DebugTargetLike { + return { + id: partial.id || 'target', + title: partial.title || '', + type: partial.type || 'page', + url: partial.url || 'http://localhost:5173/', + webSocketDebuggerUrl: partial.webSocketDebuggerUrl, + } +} + +describe('debug target helpers', () => { + it('filters out helper renderer pages like beat-sync and devtools', () => { + expect(isInspectableAiriRendererTarget(createTarget({ + url: 'http://localhost:5173/beat-sync.html', + }))).toBe(false) + + expect(isInspectableAiriRendererTarget(createTarget({ + url: 'http://localhost:5173/__inspect__', + }))).toBe(false) + + expect(isInspectableAiriRendererTarget(createTarget({ + url: 'http://localhost:5173/', + }))).toBe(true) + }) + + it('prioritizes chat and main AIRI surfaces ahead of generic pages', () => { + const targets = prioritizeInspectableAiriTargets([ + createTarget({ + id: 'generic', + title: '', + url: 'http://localhost:5173/widgets.html', + }), + createTarget({ + id: 'main', + title: 'AIRI', + url: 'http://localhost:5173/', + }), + createTarget({ + id: 'chat', + title: 'Chat', + url: 'http://localhost:5173/#/chat', + }), + ]) + + expect(targets.map(target => target.id)).toEqual(['chat', 'main', 'generic']) + }) + + it('detects chat surfaces from either target metadata or AIRI debug snapshots', () => { + expect(isChatSurfaceTarget(createTarget({ + title: 'Chat', + url: 'http://localhost:5173/', + }))).toBe(true) + + expect(isChatSurfaceTarget(createTarget({ + title: 'AIRI', + url: 'http://localhost:5173/', + }), { + route: '#/chat', + })).toBe(true) + + expect(isChatSurfaceTarget(createTarget({ + title: 'AIRI', + url: 'http://localhost:5173/', + }), { + route: '#/', + })).toBe(false) + }) +}) diff --git a/services/computer-use-mcp/src/e2e/debug-targets.ts b/services/computer-use-mcp/src/e2e/debug-targets.ts new file mode 100644 index 000000000..d4084970a --- /dev/null +++ b/services/computer-use-mcp/src/e2e/debug-targets.ts @@ -0,0 +1,70 @@ +export interface DebugTargetLike { + id: string + title: string + type: string + url: string + webSocketDebuggerUrl?: string +} + +export interface AiriDebugSnapshotLike { + route?: string + documentTitle?: string +} + +function containsFragment(input: string, fragments: string[]) { + return fragments.some(fragment => input.includes(fragment)) +} + +// NOTICE: The remote debug endpoint exposes multiple renderer pages for Electron. +// We only want actual AIRI app pages here; helper pages like beat-sync and devtools +// will never expose `window.__AIRI_DEBUG__` and should be deprioritized early. +export function isInspectableAiriRendererTarget(target: DebugTargetLike) { + if (target.type !== 'page') { + return false + } + + if (!target.url.startsWith('http://localhost:5173/')) { + return false + } + + return !containsFragment(target.url, [ + '/__inspect__', + '/__devtools__', + '/__unocss', + 'beat-sync.html', + ]) +} + +export function prioritizeInspectableAiriTargets(targets: DebugTargetLike[]) { + return [...targets] + .filter(isInspectableAiriRendererTarget) + .sort((left, right) => scoreInspectableAiriTarget(right) - scoreInspectableAiriTarget(left)) +} + +function scoreInspectableAiriTarget(target: DebugTargetLike) { + let score = 0 + + if (target.title === 'Chat') { + score += 40 + } + else if (target.title === 'AIRI') { + score += 20 + } + + if (target.url.includes('#/chat')) { + score += 30 + } + else if (target.url === 'http://localhost:5173/' || target.url === 'http://localhost:5173') { + score += 10 + } + + return score +} + +export function isChatSurfaceTarget(target: DebugTargetLike, snapshot?: AiriDebugSnapshotLike) { + if (target.title === 'Chat' || target.url.includes('#/chat')) { + return true + } + + return String(snapshot?.route || '').includes('/chat') +} diff --git a/services/computer-use-mcp/src/e2e/provider-bootstrap.test.ts b/services/computer-use-mcp/src/e2e/provider-bootstrap.test.ts new file mode 100644 index 000000000..a1fde4688 --- /dev/null +++ b/services/computer-use-mcp/src/e2e/provider-bootstrap.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' + +import { getProviderBootstrapConfig, resolveGithubModelsApiKey } from './provider-bootstrap' + +describe('provider bootstrap', () => { + it('prefers explicit github models env keys before legacy aliases', () => { + expect(resolveGithubModelsApiKey({ + processEnv: { + GITHUB_TOKEN: 'ghp-fallback', + GITHUB_MODELS_API_KEY: 'ghp-preferred', + }, + dotenvValues: {}, + })).toBe('ghp-preferred') + }) + + it('falls back to legacy GitHub_token from .env for this repo', () => { + expect(resolveGithubModelsApiKey({ + processEnv: {}, + dotenvValues: { + GitHub_token: 'ghp-legacy', + }, + })).toBe('ghp-legacy') + }) + + it('builds github-models bootstrap config with the default inference base url', () => { + expect(getProviderBootstrapConfig({ + providerId: 'github-models', + processEnv: {}, + dotenvValues: { + GitHub_token: 'ghp-legacy', + }, + })).toEqual({ + apiKey: 'ghp-legacy', + baseUrl: 'https://models.github.ai/inference', + }) + }) + + it('does not fabricate bootstrap config for unrelated providers', () => { + expect(getProviderBootstrapConfig({ + providerId: 'openrouter', + processEnv: { + GITHUB_MODELS_API_KEY: 'ghp-test', + }, + dotenvValues: {}, + })).toBeUndefined() + }) +}) diff --git a/services/computer-use-mcp/src/e2e/provider-bootstrap.ts b/services/computer-use-mcp/src/e2e/provider-bootstrap.ts new file mode 100644 index 000000000..cba66c498 --- /dev/null +++ b/services/computer-use-mcp/src/e2e/provider-bootstrap.ts @@ -0,0 +1,98 @@ +const DEFAULT_GITHUB_MODELS_BASE_URL = 'https://models.github.ai/inference' +const DEFAULT_GOOGLE_GENERATIVE_AI_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/openai/' + +const githubModelsApiKeyCandidates = [ + 'AIRI_E2E_GITHUB_MODELS_API_KEY', + 'GITHUB_MODELS_API_KEY', + 'GITHUB_TOKEN', + 'GH_TOKEN', + 'GitHub_token', +] as const + +const googleGenerativeApiKeyCandidates = [ + 'AIRI_E2E_GOOGLE_API_KEY', + 'GOOGLE_API_KEY', + 'GEMINI_API_KEY', + 'GOOGLE_GENERATIVE_AI_API_KEY', +] as const + +type EnvMap = NodeJS.ProcessEnv | Record + +function readFirstNonEmptyValue(source: EnvMap, keys: readonly string[]) { + for (const key of keys) { + const value = source[key]?.trim() + if (value) { + return value + } + } + + return '' +} + +export function resolveGithubModelsApiKey(params: { + processEnv: EnvMap + dotenvValues: Record +}) { + return readFirstNonEmptyValue(params.processEnv, githubModelsApiKeyCandidates) + || readFirstNonEmptyValue(params.dotenvValues, githubModelsApiKeyCandidates) +} + +export function resolveGoogleGenerativeApiKey(params: { + processEnv: EnvMap + dotenvValues: Record +}) { + return readFirstNonEmptyValue(params.processEnv, googleGenerativeApiKeyCandidates) + || readFirstNonEmptyValue(params.dotenvValues, googleGenerativeApiKeyCandidates) +} + +export function resolvePreferredChatProviderId(params: { + requestedProviderId?: string + processEnv: EnvMap + dotenvValues: Record +}) { + if (params.requestedProviderId?.trim()) { + return params.requestedProviderId.trim() + } + + if (resolveGithubModelsApiKey(params)) { + return 'github-models' + } + + if (resolveGoogleGenerativeApiKey(params)) { + return 'google-generative-ai' + } + + return 'github-models' +} + +export function getProviderBootstrapConfig(params: { + providerId: string + processEnv: EnvMap + dotenvValues: Record +}) { + if (params.providerId === 'github-models') { + const apiKey = resolveGithubModelsApiKey(params) + if (!apiKey) { + return undefined + } + + return { + apiKey, + baseUrl: DEFAULT_GITHUB_MODELS_BASE_URL, + } + } + + if (params.providerId === 'google-generative-ai') { + const apiKey = resolveGoogleGenerativeApiKey(params) + if (!apiKey) { + return undefined + } + + return { + apiKey, + baseUrl: DEFAULT_GOOGLE_GENERATIVE_AI_BASE_URL, + } + } + + return undefined +} diff --git a/services/computer-use-mcp/src/executors/dry-run.ts b/services/computer-use-mcp/src/executors/dry-run.ts new file mode 100644 index 000000000..6372f11d7 --- /dev/null +++ b/services/computer-use-mcp/src/executors/dry-run.ts @@ -0,0 +1,112 @@ +import type { + ClickActionInput, + ComputerUseConfig, + DesktopExecutor, + ExecutionTarget, + ExecutorActionResult, + FocusAppActionInput, + ForegroundContext, + ObserveWindowsRequest, + OpenAppActionInput, + PointerTracePoint, + PressKeysActionInput, + ScrollActionInput, + TypeTextActionInput, + WaitActionInput, + WindowObservation, +} from '../types' + +import { hostname } from 'node:os' +import { platform } from 'node:process' + +import { probeDisplayInfo, probePermissionInfo } from '../runtime-probes' +import { captureScreenshotArtifact } from '../utils/screenshot' + +async function getBestEffortForegroundContext(): Promise { + return { + available: false, + platform, + unavailableReason: 'dry-run backend does not inspect foreground window state', + } +} + +function result(notes: string[], executionTarget: ExecutionTarget): ExecutorActionResult { + return { + performed: false, + backend: 'dry-run', + notes, + executionTarget, + } +} + +function getDryRunExecutionTarget(config: ComputerUseConfig): ExecutionTarget { + return { + mode: 'dry-run', + transport: 'local', + hostName: hostname(), + sessionTag: config.sessionTag, + isolated: false, + tainted: false, + note: 'dry-run mode never injects desktop input', + } +} + +function observeWindows(request: ObserveWindowsRequest): WindowObservation { + return { + frontmostAppName: request.app, + windows: [], + observedAt: new Date().toISOString(), + } +} + +export function createDryRunExecutor(config: ComputerUseConfig): DesktopExecutor { + const executionTarget = getDryRunExecutionTarget(config) + + return { + kind: 'dry-run', + describe: () => ({ + kind: 'dry-run', + notes: [ + 'desktop input is not injected', + 'screenshots are still attempted on the current host for debugging', + ], + }), + getExecutionTarget: async () => executionTarget, + getForegroundContext: getBestEffortForegroundContext, + getDisplayInfo: () => probeDisplayInfo(config), + getPermissionInfo: () => probePermissionInfo(config), + observeWindows: async request => observeWindows(request), + takeScreenshot: request => captureScreenshotArtifact({ + label: request.label, + screenshotsDir: config.screenshotsDir, + screenshotBinary: config.binaries.screencapture, + timeoutMs: config.timeoutMs, + executionTarget, + }), + openApp: async (_input: OpenAppActionInput) => result(['dry-run: app not opened'], executionTarget), + focusApp: async (_input: FocusAppActionInput) => result(['dry-run: app not focused'], executionTarget), + click: async (_input: ClickActionInput & { pointerTrace: PointerTracePoint[] }) => result(['dry-run: click not injected'], executionTarget), + typeText: async (_input: TypeTextActionInput) => result(['dry-run: text not injected'], executionTarget), + pressKeys: async (_input: PressKeysActionInput) => result(['dry-run: shortcut not injected'], executionTarget), + scroll: async (_input: ScrollActionInput) => result(['dry-run: scroll not injected'], executionTarget), + wait: async (input: WaitActionInput) => { + await new Promise(resolve => setTimeout(resolve, Math.max(input.durationMs, 0))) + return { + performed: true, + backend: 'dry-run', + notes: ['dry-run: waited without desktop mutation'], + executionTarget, + } + }, + openTestTarget: async () => ({ + launched: true, + appName: 'dry-run-target', + windowTitle: 'Dry Run Desktop Target', + recommendedClickPoint: { + x: 180, + y: 150, + }, + executionTarget, + }), + } +} diff --git a/services/computer-use-mcp/src/executors/linux-x11.ts b/services/computer-use-mcp/src/executors/linux-x11.ts new file mode 100644 index 000000000..5a3f876ca --- /dev/null +++ b/services/computer-use-mcp/src/executors/linux-x11.ts @@ -0,0 +1,97 @@ +import type { RemoteRunnerClientOptions } from '../runner/client' +import type { + ClickActionInput, + ComputerUseConfig, + DesktopExecutor, + ForegroundContext, + PointerTracePoint, + PressKeysActionInput, + ScrollActionInput, + TypeTextActionInput, + WaitActionInput, + WindowObservation, +} from '../types' + +import { RemoteRunnerClient } from '../runner/client' +import { writeScreenshotArtifact } from '../utils/screenshot' + +export interface LinuxX11ExecutorOptions extends RemoteRunnerClientOptions { + client?: RemoteRunnerClient +} + +function unavailableContext(reason: string): ForegroundContext { + return { + available: false, + platform: 'linux', + unavailableReason: reason, + } +} + +export function createLinuxX11Executor(config: ComputerUseConfig, options: LinuxX11ExecutorOptions = {}): DesktopExecutor { + const client = options.client || new RemoteRunnerClient(config, options) + + return { + kind: 'linux-x11', + describe: () => ({ + kind: 'linux-x11', + notes: [ + 'approval, trace and audit stay on the host', + 'all desktop actions execute through a remote SSH-bound X11 runner', + ], + }), + getExecutionTarget: () => client.getExecutionTarget(), + getForegroundContext: async () => { + try { + return await client.getForegroundContext() + } + catch (error) { + return unavailableContext(error instanceof Error ? error.message : String(error)) + } + }, + getDisplayInfo: () => client.getDisplayInfo(), + getPermissionInfo: () => client.getPermissionInfo(), + observeWindows: async () => { + const context = await client.getForegroundContext() + const windows = context.available && context.appName + ? [{ + id: `${context.appName}:${context.windowTitle || 'foreground'}`, + appName: context.appName, + title: context.windowTitle, + }] + : [] + return { + frontmostAppName: context.appName, + frontmostWindowTitle: context.windowTitle, + windows, + observedAt: new Date().toISOString(), + } satisfies WindowObservation + }, + takeScreenshot: async (request) => { + const result = await client.takeScreenshot(request) + + return await writeScreenshotArtifact({ + label: request.label, + screenshotsDir: config.screenshotsDir, + dataBase64: result.dataBase64, + publicUrl: result.publicUrl, + note: result.note, + executionTarget: result.executionTarget, + }) + }, + openApp: async () => { + throw new Error('linux-x11 executor does not implement app.open in this v1') + }, + focusApp: async () => { + throw new Error('linux-x11 executor does not implement app.focus in this v1') + }, + click: async (input: ClickActionInput & { pointerTrace: PointerTracePoint[] }) => await client.click(input), + typeText: async (input: TypeTextActionInput) => await client.typeText(input), + pressKeys: async (input: PressKeysActionInput) => await client.pressKeys(input), + scroll: async (input: ScrollActionInput) => await client.scroll(input), + wait: async (input: WaitActionInput) => await client.wait(input), + openTestTarget: async () => await client.openTestTarget(), + close: async () => { + await client.close() + }, + } +} diff --git a/services/computer-use-mcp/src/executors/macos-local.ts b/services/computer-use-mcp/src/executors/macos-local.ts new file mode 100644 index 000000000..59a477877 --- /dev/null +++ b/services/computer-use-mcp/src/executors/macos-local.ts @@ -0,0 +1,557 @@ +import type { + ClickActionInput, + ComputerUseConfig, + DesktopExecutor, + ExecutionTarget, + ExecutorActionResult, + FocusAppActionInput, + ForegroundContext, + ObserveWindowsRequest, + OpenAppActionInput, + PointerTracePoint, + PressKeysActionInput, + ScrollActionInput, + TypeTextActionInput, + WaitActionInput, + WindowObservation, +} from '../types' + +import { existsSync, readdirSync } from 'node:fs' +import { hostname } from 'node:os' +import { join } from 'node:path' +import { platform } from 'node:process' + +import { appNamesMatch, getKnownAppLaunchNames } from '../app-aliases' +import { probeDisplayInfo, probePermissionInfo } from '../runtime-probes' +import { runProcess } from '../utils/process' +import { captureScreenshotArtifact } from '../utils/screenshot' +import { runSwiftScript } from '../utils/swift' + +const buttonNames = { + left: 0, + right: 1, + middle: 2, +} as const + +const keyCodeMap: Record = { + a: 0, + b: 11, + c: 8, + d: 2, + e: 14, + f: 3, + g: 5, + h: 4, + i: 34, + j: 38, + k: 40, + l: 37, + m: 46, + n: 45, + o: 31, + p: 35, + q: 12, + r: 15, + s: 1, + t: 17, + u: 32, + v: 9, + w: 13, + x: 7, + y: 16, + z: 6, + 0: 29, + 1: 18, + 2: 19, + 3: 20, + 4: 21, + 5: 23, + 6: 22, + 7: 26, + 8: 28, + 9: 25, + enter: 36, + return: 36, + tab: 48, + space: 49, + escape: 53, + esc: 53, + delete: 51, + backspace: 51, + up: 126, + down: 125, + left: 123, + right: 124, +} + +const modifierFlags: Record = { + command: '.maskCommand', + cmd: '.maskCommand', + shift: '.maskShift', + control: '.maskControl', + ctrl: '.maskControl', + option: '.maskAlternate', + alt: '.maskAlternate', +} + +function createExecutionTarget(config: ComputerUseConfig): ExecutionTarget { + return { + mode: 'local-windowed', + transport: 'local', + hostName: hostname(), + sessionTag: config.sessionTag, + isolated: false, + tainted: false, + note: 'local macOS window automation via Swift + Quartz', + } +} + +function result(notes: string[], executionTarget: ExecutionTarget): ExecutorActionResult { + return { + performed: true, + backend: 'macos-local', + notes, + executionTarget, + } +} + +function fallbackContext(reason: string): ForegroundContext { + return { + available: false, + platform, + unavailableReason: reason, + } +} + +async function runMacOsJsonScript(config: ComputerUseConfig, source: string, stdinPayload?: unknown): Promise { + const { stdout } = await runSwiftScript({ + swiftBinary: config.binaries.swift, + timeoutMs: config.timeoutMs, + source, + stdinPayload, + }) + + return JSON.parse(stdout.trim()) as T +} + +function observeWindowsScript() { + return String.raw` +import AppKit +import CoreGraphics +import Foundation + +func boundsDict(_ value: NSDictionary?) -> [String: Int]? { + guard let value else { return nil } + var rect = CGRect.zero + guard CGRectMakeWithDictionaryRepresentation(value, &rect) else { return nil } + return [ + "x": Int(rect.origin.x.rounded()), + "y": Int(rect.origin.y.rounded()), + "width": Int(rect.size.width.rounded()), + "height": Int(rect.size.height.rounded()) + ] +} + +let environment = ProcessInfo.processInfo.environment +let rawInput = environment["COMPUTER_USE_SWIFT_STDIN"] ?? "{}" +let inputData = rawInput.data(using: .utf8) ?? Data() +let input = (try? JSONSerialization.jsonObject(with: inputData)) as? [String: Any] ?? [:] +let limit = (input["limit"] as? Int) ?? 12 +let appFilter = ((input["app"] as? String) ?? "").lowercased().trimmingCharacters(in: .whitespacesAndNewlines) +let frontmostAppName = NSWorkspace.shared.frontmostApplication?.localizedName + +let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements] +let rawWindowInfo = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] ?? [] +var windows: [[String: Any]] = [] +for window in rawWindowInfo { + let ownerName = (window[kCGWindowOwnerName as String] as? String) ?? "Unknown" + if !appFilter.isEmpty && !ownerName.lowercased().contains(appFilter) { + continue + } + + let alpha = window[kCGWindowAlpha as String] as? Double ?? 1.0 + let layer = window[kCGWindowLayer as String] as? Int ?? 0 + let bounds = boundsDict(window[kCGWindowBounds as String] as? NSDictionary) + let title = (window[kCGWindowName as String] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + let ownerPid = window[kCGWindowOwnerPID as String] as? Int ?? 0 + + if alpha <= 0 || (bounds?["width"] ?? 0) <= 1 || (bounds?["height"] ?? 0) <= 1 { + continue + } + + windows.append([ + "id": "\(ownerPid):\(layer):\(title ?? ownerName)", + "appName": ownerName, + "title": title as Any, + "bounds": bounds as Any, + "ownerPid": ownerPid, + "layer": layer, + "isOnScreen": true, + ]) + + if windows.count >= limit { + break + } +} + +let frontmostWindowTitle = windows.first(where: { ($0["appName"] as? String) == frontmostAppName })?["title"] +let payload: [String: Any] = [ + "frontmostAppName": frontmostAppName as Any, + "frontmostWindowTitle": frontmostWindowTitle as Any, + "windows": windows, + "observedAt": ISO8601DateFormatter().string(from: Date()), +] + +let data = try JSONSerialization.data(withJSONObject: payload, options: []) +print(String(data: data, encoding: .utf8)!) +` +} + +function moveAndClickScript() { + return String.raw` +import CoreGraphics +import Foundation + +func mouseButton(_ value: Int) -> CGMouseButton { + switch value { + case 1: return .right + case 2: return .center + default: return .left + } +} + +func mouseDownType(_ button: CGMouseButton) -> CGEventType { + switch button { + case .right: return .rightMouseDown + case .center: return .otherMouseDown + default: return .leftMouseDown + } +} + +func mouseUpType(_ button: CGMouseButton) -> CGEventType { + switch button { + case .right: return .rightMouseUp + case .center: return .otherMouseUp + default: return .leftMouseUp + } +} + +let environment = ProcessInfo.processInfo.environment +let rawInput = environment["COMPUTER_USE_SWIFT_STDIN"] ?? "{}" +let inputData = rawInput.data(using: .utf8) ?? Data() +let input = (try? JSONSerialization.jsonObject(with: inputData)) as? [String: Any] ?? [:] +let trace = input["pointerTrace"] as? [[String: Any]] ?? [] +let buttonRaw = input["button"] as? Int ?? 0 +let clickCount = input["clickCount"] as? Int ?? 1 +let button = mouseButton(buttonRaw) + +for point in trace { + let x = point["x"] as? Double ?? 0 + let y = point["y"] as? Double ?? 0 + let delayMs = point["delayMs"] as? Int ?? 0 + let location = CGPoint(x: x, y: y) + if let moveEvent = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: location, mouseButton: .left) { + moveEvent.post(tap: .cghidEventTap) + } + if delayMs > 0 { + usleep(useconds_t(delayMs * 1000)) + } +} + +let lastPoint = trace.last +let x = lastPoint?["x"] as? Double ?? 0 +let y = lastPoint?["y"] as? Double ?? 0 +let location = CGPoint(x: x, y: y) + +for _ in 0.. 0 else { return } + chars.withUnsafeBufferPointer { buffer in + if let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) { + keyDown.keyboardSetUnicodeString(stringLength: length, unicodeString: buffer.baseAddress!) + keyUp.keyboardSetUnicodeString(stringLength: length, unicodeString: buffer.baseAddress!) + keyDown.post(tap: .cghidEventTap) + keyUp.post(tap: .cghidEventTap) + } + } + + // NOTICE: Electron/Vue textareas can drop tail characters when a burst of + // Quartz keyboard events is posted back-to-back with no pacing. A short + // delay between Unicode events keeps the renderer input queue stable enough + // for end-to-end desktop automation. + usleep(characterDelayMicros) +} + +for character in text { + postText(String(character)) +} + +if !text.isEmpty { + usleep(settleDelayMicros) +} + +if pressEnter { + if let down = CGEvent(keyboardEventSource: nil, virtualKey: 36, keyDown: true), + let up = CGEvent(keyboardEventSource: nil, virtualKey: 36, keyDown: false) { + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + } +} + +print("{}") +` +} + +function pressKeysScript(mainKeyCode: number, modifierMaskExpr: string) { + return String.raw` +import CoreGraphics +import Foundation + +let keyCode: CGKeyCode = ${mainKeyCode} +let modifierFlags: CGEventFlags = ${modifierMaskExpr} + +if let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: false) { + keyDown.flags = modifierFlags + keyUp.flags = modifierFlags + keyDown.post(tap: .cghidEventTap) + keyUp.post(tap: .cghidEventTap) +} + +print("{}") +` +} + +function scrollScript() { + return String.raw` +import CoreGraphics +import Foundation + +let environment = ProcessInfo.processInfo.environment +let rawInput = environment["COMPUTER_USE_SWIFT_STDIN"] ?? "{}" +let inputData = rawInput.data(using: .utf8) ?? Data() +let input = (try? JSONSerialization.jsonObject(with: inputData)) as? [String: Any] ?? [:] +let x = input["x"] as? Double +let y = input["y"] as? Double +let deltaX = Int32(input["deltaX"] as? Double ?? 0) +let deltaY = Int32(input["deltaY"] as? Double ?? 0) + +if let x, let y { + let location = CGPoint(x: x, y: y) + if let moveEvent = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: location, mouseButton: .left) { + moveEvent.post(tap: .cghidEventTap) + } +} + +if let scrollEvent = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: deltaY, wheel2: deltaX, wheel3: 0) { + scrollEvent.post(tap: .cghidEventTap) +} + +print("{}") +` +} + +async function observeWindows(config: ComputerUseConfig, request: ObserveWindowsRequest): Promise { + return await runMacOsJsonScript(config, observeWindowsScript(), request) +} + +function observationToForegroundContext(observation: WindowObservation): ForegroundContext { + const frontmost = observation.windows.find(window => window.appName === observation.frontmostAppName) + return { + available: Boolean(observation.frontmostAppName), + appName: observation.frontmostAppName, + windowTitle: observation.frontmostWindowTitle, + windowBounds: frontmost?.bounds, + platform, + unavailableReason: observation.frontmostAppName ? undefined : 'frontmost application unavailable', + } +} + +async function ensureMacOS() { + if (platform !== 'darwin') { + throw new Error('macos-local executor requires macOS') + } +} + +function resolveInstalledMacAppName(app: string) { + const searchRoots = [ + '/Applications', + join(process.env.HOME || '', 'Applications'), + ].filter(Boolean) + + for (const root of searchRoots) { + if (!existsSync(root)) { + continue + } + + const appBundle = readdirSync(root).find((entry) => { + if (!entry.endsWith('.app')) { + return false + } + + const bundleName = entry.replace(/\.app$/u, '') + return getKnownAppLaunchNames(app).some(candidate => appNamesMatch(bundleName, candidate)) + }) + + if (appBundle) { + return appBundle.replace(/\.app$/u, '') + } + } + + return app +} + +async function runOpenCommand(config: ComputerUseConfig, app: string) { + await runProcess(config.binaries.open, ['-a', resolveInstalledMacAppName(app)], { + timeoutMs: config.timeoutMs, + }) +} + +async function activateApp(config: ComputerUseConfig, app: string) { + const resolvedApp = resolveInstalledMacAppName(app) + await runProcess(config.binaries.osascript, [ + '-e', + `tell application ${JSON.stringify(resolvedApp)} to activate`, + ], { + timeoutMs: config.timeoutMs, + }) +} + +export function createMacOSLocalExecutor(config: ComputerUseConfig): DesktopExecutor { + const executionTarget = createExecutionTarget(config) + + return { + kind: 'macos-local', + describe: () => ({ + kind: 'macos-local', + notes: [ + 'desktop actions run on the current macOS host', + 'window observation uses NSWorkspace + CGWindowList', + 'input injection uses Swift + Quartz CGEvent', + ], + }), + getExecutionTarget: async () => executionTarget, + getForegroundContext: async () => { + try { + await ensureMacOS() + return observationToForegroundContext(await observeWindows(config, { limit: 8 })) + } + catch (error) { + return fallbackContext(error instanceof Error ? error.message : String(error)) + } + }, + getDisplayInfo: () => probeDisplayInfo(config), + getPermissionInfo: () => probePermissionInfo(config), + observeWindows: async (request) => { + await ensureMacOS() + return await observeWindows(config, request) + }, + takeScreenshot: request => captureScreenshotArtifact({ + label: request.label, + screenshotsDir: config.screenshotsDir, + screenshotBinary: config.binaries.screencapture, + timeoutMs: config.timeoutMs, + executionTarget, + }), + openApp: async (input: OpenAppActionInput) => { + await ensureMacOS() + await runOpenCommand(config, input.app) + return result([`opened app ${input.app}`], executionTarget) + }, + focusApp: async (input: FocusAppActionInput) => { + await ensureMacOS() + await runOpenCommand(config, input.app) + await activateApp(config, input.app) + return result([`focused app ${input.app}`], executionTarget) + }, + click: async (input: ClickActionInput & { pointerTrace: PointerTracePoint[] }) => { + await ensureMacOS() + await runMacOsJsonScript>(config, moveAndClickScript(), { + pointerTrace: input.pointerTrace, + button: buttonNames[input.button || 'left'], + clickCount: input.clickCount ?? 1, + }) + return { + ...result(['clicked on local macOS desktop'], executionTarget), + pointerTrace: input.pointerTrace, + } + }, + typeText: async (input: TypeTextActionInput) => { + await ensureMacOS() + await runMacOsJsonScript>(config, typeTextScript(), { + text: input.text, + pressEnter: input.pressEnter ?? false, + }) + return result(['typed text on local macOS desktop'], executionTarget) + }, + pressKeys: async (input: PressKeysActionInput) => { + await ensureMacOS() + const normalized = input.keys.map(key => key.trim().toLowerCase()).filter(Boolean) + if (normalized.length === 0) + throw new Error('press_keys requires at least one key') + + const mainKey = normalized.at(-1)! + const keyCode = keyCodeMap[mainKey] + if (typeof keyCode !== 'number') { + throw new TypeError(`unsupported macOS key for press_keys: ${mainKey}`) + } + + const modifiers = normalized.slice(0, -1) + const modifierMaskExpr = modifiers.length > 0 + ? modifiers.map((modifier) => { + const flag = modifierFlags[modifier] + if (!flag) + throw new Error(`unsupported modifier key: ${modifier}`) + return flag + }).join(' | ') + : '[]' + + await runMacOsJsonScript>(config, pressKeysScript(keyCode, modifierMaskExpr), {}) + return result([`pressed keys ${normalized.join('+')}`], executionTarget) + }, + scroll: async (input: ScrollActionInput) => { + await ensureMacOS() + await runMacOsJsonScript>(config, scrollScript(), { + x: input.x, + y: input.y, + deltaX: input.deltaX ?? 0, + deltaY: input.deltaY, + }) + return result(['scrolled on local macOS desktop'], executionTarget) + }, + wait: async (input: WaitActionInput) => { + await new Promise(resolve => setTimeout(resolve, Math.max(input.durationMs, 0))) + return result(['waited on local macOS desktop'], executionTarget) + }, + } +} diff --git a/services/computer-use-mcp/src/index.ts b/services/computer-use-mcp/src/index.ts new file mode 100644 index 000000000..17f0144fc --- /dev/null +++ b/services/computer-use-mcp/src/index.ts @@ -0,0 +1,15 @@ +export { resolveComputerUseConfig } from './config' +export { createComputerUseMcpServer, startComputerUseMcpServer } from './server' +export { RunStateManager } from './state' +export type { ActiveTask, RunState, TaskPhase, TaskStep } from './state' +export { buildRecoveryPlan, evaluateStrategy, summarizeAdvisories } from './strategy' +export type { AdvisoryKind, StrategyAdvisory } from './strategy' +export type * from './types' +export { + createAppBrowseAndActWorkflow, + createDevInspectFailureWorkflow, + createDevRunTestsWorkflow, + executeWorkflow, + resumeWorkflow, +} from './workflows' +export type { WorkflowDefinition, WorkflowExecutionResult, WorkflowSuspension } from './workflows' diff --git a/services/computer-use-mcp/src/policy.test.ts b/services/computer-use-mcp/src/policy.test.ts new file mode 100644 index 000000000..d67208782 --- /dev/null +++ b/services/computer-use-mcp/src/policy.test.ts @@ -0,0 +1,203 @@ +import type { ComputerUseConfig } from './types' + +import { describe, expect, it } from 'vitest' + +import { evaluateActionPolicy } from './policy' +import { createTestConfig } from './test-fixtures' + +const baseConfig: ComputerUseConfig = createTestConfig({ + executor: 'dry-run', + permissionChainHint: 'Terminal -> local dry-run', + denyApps: ['airi'], + requireAllowedBoundsForMutatingActions: false, + requireCoordinateAlignmentForMutatingActions: false, + requireSessionTagForMutatingActions: false, +}) + +describe('evaluateActionPolicy', () => { + it('requires approval for mutating ui actions in actions mode', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'click', + input: { + x: 10, + y: 12, + }, + }, + config: baseConfig, + context: { + available: true, + appName: 'Finder', + platform: 'darwin', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(true) + expect(decision.requiresApproval).toBe(true) + }) + + it('requires approval for terminal execution in actions mode', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'terminal_exec', + input: { + command: 'pwd', + }, + }, + config: { + ...baseConfig, + approvalMode: 'actions', + }, + context: { + available: false, + platform: 'darwin', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(true) + expect(decision.requiresApproval).toBe(true) + expect(decision.riskLevel).toBe('high') + }) + + it('skips approval for terminal execution in never mode', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'terminal_exec', + input: { + command: 'pwd', + }, + }, + config: { + ...baseConfig, + approvalMode: 'never', + }, + context: { + available: false, + platform: 'darwin', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(true) + expect(decision.requiresApproval).toBe(false) + expect(decision.riskLevel).toBe('high') + }) + + it('treats secret env reads as high-risk but non-mutating', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'secret_read_env_value', + input: { + filePath: '/Users/liuziheng/airi/.env', + keys: ['DISCORD_BOT_TOKEN'], + }, + }, + config: { + ...baseConfig, + approvalMode: 'actions', + }, + context: { + available: false, + platform: 'darwin', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(true) + expect(decision.requiresApproval).toBe(true) + expect(decision.riskLevel).toBe('high') + }) + + it('denies sensitive foreground apps for ui actions', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'press_keys', + input: { + keys: ['command', 'l'], + }, + }, + config: baseConfig, + context: { + available: true, + appName: 'AIRI', + platform: 'darwin', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(false) + expect(decision.reasons[0]).toContain('foreground app denied') + }) + + it('denies opening apps outside the configured openable list', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'open_app', + input: { + app: 'Safari', + }, + }, + config: baseConfig, + context: { + available: false, + platform: 'darwin', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(false) + expect(decision.reasons[0]).toContain('COMPUTER_USE_OPENABLE_APPS') + }) + + it('allows app aliases when the canonical app is configured', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'open_app', + input: { + app: 'VS Code', + }, + }, + config: baseConfig, + context: { + available: false, + platform: 'darwin', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(true) + }) + + it('denies app actions on the legacy linux-x11 executor', () => { + const decision = evaluateActionPolicy({ + action: { + kind: 'focus_app', + input: { + app: 'Terminal', + }, + }, + config: { + ...baseConfig, + executor: 'linux-x11', + }, + context: { + available: false, + platform: 'linux', + }, + operationsExecuted: 0, + operationUnitsConsumed: 0, + }) + + expect(decision.allowed).toBe(false) + expect(decision.reasons[0]).toContain('linux-x11 executor does not support app open/focus actions') + }) +}) diff --git a/services/computer-use-mcp/src/policy.ts b/services/computer-use-mcp/src/policy.ts new file mode 100644 index 000000000..5625e2dfb --- /dev/null +++ b/services/computer-use-mcp/src/policy.ts @@ -0,0 +1,225 @@ +import type { ActionInvocation, ComputerUseConfig, ForegroundContext, PolicyDecision } from './types' + +import { resolveConfiguredOpenableApp } from './app-aliases' + +function includesPattern(value: string | undefined, patterns: string[]) { + const normalizedValue = value?.trim().toLowerCase() + if (!normalizedValue) + return false + + return patterns.some(pattern => normalizedValue.includes(pattern.toLowerCase())) +} + +function isMutatingAction(action: ActionInvocation) { + return !['screenshot', 'observe_windows', 'wait', 'terminal_reset', 'clipboard_read_text', 'secret_read_env_value'].includes(action.kind) +} + +function isUiInteractionAction(action: ActionInvocation) { + return ['click', 'type_text', 'press_keys', 'scroll', 'open_app', 'focus_app'].includes(action.kind) +} + +function getCoordinate(action: ActionInvocation) { + switch (action.kind) { + case 'click': + return { x: action.input.x, y: action.input.y } + case 'type_text': + if (typeof action.input.x === 'number' && typeof action.input.y === 'number') { + return { x: action.input.x, y: action.input.y } + } + return undefined + case 'scroll': + if (typeof action.input.x === 'number' && typeof action.input.y === 'number') { + return { x: action.input.x, y: action.input.y } + } + return undefined + default: + return undefined + } +} + +function estimateOperationUnits(action: ActionInvocation) { + switch (action.kind) { + case 'screenshot': + return 3 + case 'observe_windows': + return 1 + case 'open_app': + case 'focus_app': + return 2 + case 'clipboard_read_text': + case 'secret_read_env_value': + return 1 + case 'clipboard_write_text': + return Math.max(2, Math.ceil(action.input.text.length / 64)) + case 'click': + return 1 + case 'type_text': + return Math.max(2, Math.ceil(action.input.text.length / 48)) + case 'press_keys': + return 1 + case 'scroll': + return 1 + case 'wait': + return 1 + case 'terminal_exec': + return Math.max(4, Math.ceil(action.input.command.length / 48)) + case 'terminal_reset': + return 1 + } +} + +// NOTICE: Key aliases must be normalised to canonical names so the +// denied-shortcuts set matches regardless of how the caller spells them. +// See: macOS modifier naming conventions. +const keyAliases: Record = { + cmd: 'command', + meta: 'command', + opt: 'option', + ctrl: 'control', +} + +function normalizeShortcut(keys: string[]) { + return keys + .map((key) => { + const lower = key.trim().toLowerCase() + return keyAliases[lower] ?? lower + }) + .sort() + .join('+') +} + +const deniedShortcuts = new Set([ + 'command+q', + 'command+space', + 'command+tab', + 'alt+tab', + 'option+tab', +]) + +export function evaluateActionPolicy(params: { + action: ActionInvocation + config: ComputerUseConfig + context: ForegroundContext + operationsExecuted: number + operationUnitsConsumed: number +}): PolicyDecision { + const reasons: string[] = [] + const estimatedOperationUnits = estimateOperationUnits(params.action) + const mutating = isMutatingAction(params.action) + let allowed = true + let requiresApproval = false + let riskLevel: PolicyDecision['riskLevel'] = 'low' + + if (params.operationsExecuted >= params.config.maxOperations) { + reasons.push('session operation budget exhausted') + allowed = false + } + + if ((params.operationUnitsConsumed + estimatedOperationUnits) > params.config.maxOperationUnits) { + reasons.push('session operation-unit budget exhausted') + allowed = false + } + + const coordinate = getCoordinate(params.action) + if (coordinate && params.config.allowedBounds) { + const { x, y } = coordinate + const { allowedBounds } = params.config + const withinBounds = x >= allowedBounds.x + && y >= allowedBounds.y + && x <= (allowedBounds.x + allowedBounds.width) + && y <= (allowedBounds.y + allowedBounds.height) + + if (!withinBounds) { + reasons.push('requested coordinate is outside the allowed bounds') + allowed = false + } + } + + if (isUiInteractionAction(params.action) && params.context.available) { + if (includesPattern(params.context.appName, params.config.denyApps)) { + reasons.push(`foreground app denied: ${params.context.appName}`) + allowed = false + } + + if (includesPattern(params.context.windowTitle, params.config.denyWindowTitles)) { + reasons.push(`foreground window denied: ${params.context.windowTitle}`) + allowed = false + } + } + else if (mutating && isUiInteractionAction(params.action) && !params.context.available && params.action.kind !== 'open_app' && params.action.kind !== 'focus_app') { + reasons.push(`foreground context unavailable: ${params.context.unavailableReason || 'unknown reason'}`) + requiresApproval = params.config.approvalMode !== 'never' + } + + if (params.action.kind === 'open_app' || params.action.kind === 'focus_app') { + if (params.config.executor === 'linux-x11') { + reasons.push('linux-x11 executor does not support app open/focus actions in this legacy path') + allowed = false + } + + const resolvedApp = resolveConfiguredOpenableApp(params.action.input.app, params.config.openableApps) + if (!resolvedApp) { + reasons.push(`app is not in COMPUTER_USE_OPENABLE_APPS: ${params.action.input.app}`) + allowed = false + } + if (includesPattern(resolvedApp || params.action.input.app, params.config.denyApps)) { + reasons.push(`app denied by policy: ${resolvedApp || params.action.input.app}`) + allowed = false + } + requiresApproval = true + riskLevel = 'medium' + } + + if (params.action.kind === 'press_keys') { + const shortcut = normalizeShortcut(params.action.input.keys) + if (deniedShortcuts.has(shortcut)) { + reasons.push(`shortcut denied by default policy: ${shortcut}`) + allowed = false + } + } + + if (params.action.kind === 'type_text' && params.action.input.text.length > 160) { + reasons.push('typing a long payload should be reviewed') + requiresApproval = true + riskLevel = 'high' + } + + if (params.action.kind === 'terminal_exec') { + requiresApproval = true + riskLevel = 'high' + } + + if (params.action.kind === 'clipboard_read_text' || params.action.kind === 'clipboard_write_text' || params.action.kind === 'secret_read_env_value') { + requiresApproval = true + riskLevel = 'high' + } + + if (params.action.kind === 'click' || params.action.kind === 'press_keys' || params.action.kind === 'scroll') { + riskLevel = 'medium' + } + + if (params.action.kind === 'type_text') { + riskLevel = 'high' + } + + if (params.config.approvalMode === 'never') { + // NOTICE: 'never' mode overrides all per-action approval flags. + // This is intentional for automated/demo scenarios. + requiresApproval = false + } + else if (params.config.approvalMode === 'all') { + requiresApproval = true + } + else if (params.config.approvalMode === 'actions' && mutating && params.action.kind !== 'terminal_exec' && params.action.kind !== 'open_app' && params.action.kind !== 'focus_app') { + requiresApproval = true + } + + return { + allowed, + requiresApproval, + reason: reasons[0], + reasons, + riskLevel, + estimatedOperationUnits, + } +} diff --git a/services/computer-use-mcp/src/preflight.test.ts b/services/computer-use-mcp/src/preflight.test.ts new file mode 100644 index 000000000..55ace9a43 --- /dev/null +++ b/services/computer-use-mcp/src/preflight.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' + +import { getRuntimePreflight } from './preflight' +import { + createDisplayInfo, + createLastScreenshot, + createLocalExecutionTarget, + createRemoteExecutionTarget, + createTestConfig, +} from './test-fixtures' + +describe('getRuntimePreflight', () => { + it('denies non-remote execution for linux-x11 tools', () => { + const preflight = getRuntimePreflight({ + config: createTestConfig(), + lastScreenshot: createLastScreenshot(), + displayInfo: createDisplayInfo(), + executionTarget: createRemoteExecutionTarget({ + mode: 'dry-run', + transport: 'local', + }), + }) + + expect(preflight.blockingIssues).toContain('desktop tools require a remote linux-x11 execution target') + }) + + it('denies mismatched session tags', () => { + const preflight = getRuntimePreflight({ + config: createTestConfig(), + lastScreenshot: createLastScreenshot(), + displayInfo: createDisplayInfo(), + executionTarget: createRemoteExecutionTarget({ + sessionTag: 'different-session', + }), + }) + + expect(preflight.blockingIssues[0]).toContain('does not match expected') + }) + + it('denies display mismatches against allowed bounds', () => { + const preflight = getRuntimePreflight({ + config: createTestConfig(), + lastScreenshot: createLastScreenshot(), + displayInfo: createDisplayInfo({ + logicalWidth: 1440, + logicalHeight: 900, + }), + executionTarget: createRemoteExecutionTarget(), + }) + + expect(preflight.blockingIssues[0]).toContain('does not match allowed bounds 1280x720') + }) + + it('requires a fresh screenshot after the runner is tainted', () => { + const preflight = getRuntimePreflight({ + config: createTestConfig(), + lastScreenshot: createLastScreenshot(), + displayInfo: createDisplayInfo(), + executionTarget: createRemoteExecutionTarget({ + tainted: true, + note: 'ssh transport closed unexpectedly', + }), + }) + + expect(preflight.mutationReadinessIssues).toContain('remote runner session is tainted; capture a fresh screenshot before resuming mutations') + }) + + it('allows macos-local execution without remote binding checks', () => { + const preflight = getRuntimePreflight({ + config: createTestConfig({ + executor: 'macos-local', + requireAllowedBoundsForMutatingActions: false, + requireCoordinateAlignmentForMutatingActions: false, + requireSessionTagForMutatingActions: false, + }), + displayInfo: createDisplayInfo({ + platform: 'darwin', + }), + executionTarget: createLocalExecutionTarget(), + }) + + expect(preflight.blockingIssues).toEqual([]) + expect(preflight.mutationReadinessIssues).toEqual([]) + }) +}) diff --git a/services/computer-use-mcp/src/preflight.ts b/services/computer-use-mcp/src/preflight.ts new file mode 100644 index 000000000..6fd2e0d73 --- /dev/null +++ b/services/computer-use-mcp/src/preflight.ts @@ -0,0 +1,109 @@ +import type { + ComputerUseConfig, + CoordinateSpaceInfo, + DisplayInfo, + ExecutionTarget, + LastScreenshotInfo, + LaunchContext, +} from './types' + +import { buildCoordinateSpaceInfo, resolveLaunchContext } from './runtime-probes' + +function getScreenshotBindingIssue(params: { + config: ComputerUseConfig + lastScreenshot?: LastScreenshotInfo + executionTarget: ExecutionTarget +}) { + if (!params.lastScreenshot) { + return 'capture a fresh screenshot before mutating the remote desktop' + } + + if (params.executionTarget.mode !== params.lastScreenshot.executionTargetMode) { + return 'the latest screenshot was captured from a different execution target mode' + } + + if (params.executionTarget.hostName !== params.lastScreenshot.sourceHostName) { + return 'the latest screenshot was captured on a different remote host' + } + + if (params.executionTarget.displayId !== params.lastScreenshot.sourceDisplayId) { + return 'the latest screenshot was captured from a different remote display' + } + + if (params.config.sessionTag && params.config.sessionTag !== params.lastScreenshot.sourceSessionTag) { + return 'the latest screenshot was captured from a different remote session tag' + } + + return undefined +} + +export interface RuntimePreflight { + launchContext: LaunchContext + coordinateSpace: CoordinateSpaceInfo + blockingIssues: string[] + mutationReadinessIssues: string[] +} + +export function getRuntimePreflight(params: { + config: ComputerUseConfig + lastScreenshot?: LastScreenshotInfo + displayInfo: DisplayInfo + executionTarget: ExecutionTarget +}): RuntimePreflight { + const launchContext = resolveLaunchContext(params.config) + const coordinateSpace = buildCoordinateSpaceInfo({ + config: params.config, + lastScreenshot: params.lastScreenshot, + displayInfo: params.displayInfo, + }) + + const blockingIssues: string[] = [] + if (params.config.executor === 'linux-x11') { + if (params.executionTarget.mode !== 'remote') { + blockingIssues.push('desktop tools require a remote linux-x11 execution target') + } + + if (params.config.requireSessionTagForMutatingActions && !params.config.sessionTag) { + blockingIssues.push('COMPUTER_USE_SESSION_TAG is required before remote execution is allowed') + } + else if (params.config.sessionTag && params.executionTarget.sessionTag !== params.config.sessionTag) { + blockingIssues.push(`remote sessionTag ${params.executionTarget.sessionTag || 'missing'} does not match expected ${params.config.sessionTag}`) + } + + if (params.config.requireAllowedBoundsForMutatingActions && !params.config.allowedBounds) { + blockingIssues.push('COMPUTER_USE_ALLOWED_BOUNDS must be configured before remote execution is allowed') + } + else if (params.config.allowedBounds && params.displayInfo.available) { + if (params.displayInfo.logicalWidth !== params.config.allowedBounds.width || params.displayInfo.logicalHeight !== params.config.allowedBounds.height) { + blockingIssues.push(`remote display ${params.displayInfo.logicalWidth || '?'}x${params.displayInfo.logicalHeight || '?'} does not match allowed bounds ${params.config.allowedBounds.width}x${params.config.allowedBounds.height}`) + } + } + } + + const mutationReadinessIssues = [...blockingIssues] + if (params.config.executor === 'linux-x11') { + if (params.executionTarget.tainted) { + mutationReadinessIssues.push('remote runner session is tainted; capture a fresh screenshot before resuming mutations') + } + + const screenshotBindingIssue = getScreenshotBindingIssue({ + config: params.config, + lastScreenshot: params.lastScreenshot, + executionTarget: params.executionTarget, + }) + if (screenshotBindingIssue) { + mutationReadinessIssues.push(screenshotBindingIssue) + } + + if (params.config.requireCoordinateAlignmentForMutatingActions && !coordinateSpace.readyForMutations) { + mutationReadinessIssues.push(coordinateSpace.reason) + } + } + + return { + launchContext, + coordinateSpace, + blockingIssues, + mutationReadinessIssues, + } +} diff --git a/services/computer-use-mcp/src/remote/ssh.ts b/services/computer-use-mcp/src/remote/ssh.ts new file mode 100644 index 000000000..737f011f1 --- /dev/null +++ b/services/computer-use-mcp/src/remote/ssh.ts @@ -0,0 +1,165 @@ +import type { ComputerUseConfig } from '../types' + +import { spawn } from 'node:child_process' + +import { runProcess } from '../utils/process' + +function quoteShell(value: string) { + return `'${value.replaceAll(`'`, `'\\''`)}'` +} + +function buildSshTarget(config: ComputerUseConfig) { + if (!config.remoteSshHost || !config.remoteSshUser) { + throw new Error('remote linux-x11 execution requires COMPUTER_USE_REMOTE_SSH_HOST and COMPUTER_USE_REMOTE_SSH_USER') + } + + return `${config.remoteSshUser}@${config.remoteSshHost}` +} + +export function normalizeRemoteShellPath(value: string) { + return value.replace(/^~(?=\/|$)/, '${HOME}') +} + +export function buildRemoteShellCommandArgs(config: ComputerUseConfig, command: string) { + return [ + '-T', + '-p', + String(config.remoteSshPort), + '-o', + 'BatchMode=yes', + '-o', + 'ServerAliveInterval=30', + '-o', + 'ServerAliveCountMax=3', + buildSshTarget(config), + 'sh', + '-lc', + quoteShell(command), + ] +} + +export async function runRemoteCommand(config: ComputerUseConfig, command: string, options: { + timeoutMs?: number + stdin?: string +} = {}) { + return await runProcess(config.binaries.ssh, buildRemoteShellCommandArgs(config, command), { + timeoutMs: options.timeoutMs, + stdin: options.stdin, + env: process.env, + }) +} + +export async function uploadDirectoryToRemote(config: ComputerUseConfig, params: { + sourceDir: string + remoteDir: string + timeoutMs?: number +}) { + const remoteDir = normalizeRemoteShellPath(params.remoteDir) + const timeoutMs = params.timeoutMs ?? 120_000 + + await new Promise((resolve, reject) => { + const tar = spawn(config.binaries.tar, ['-czf', '-', '-C', params.sourceDir, '.'], { + env: { + ...process.env, + COPYFILE_DISABLE: '1', + COPY_EXTENDED_ATTRIBUTES_DISABLE: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + const ssh = spawn(config.binaries.ssh, buildRemoteShellCommandArgs( + config, + `rm -rf ${remoteDir} && mkdir -p ${remoteDir} && tar -xzf - -C ${remoteDir} && find ${remoteDir} -name '._*' -delete`, + ), { + env: process.env, + stdio: ['pipe', 'pipe', 'pipe'], + }) + + let tarStderr = '' + let sshStderr = '' + let finished = false + + const timer = setTimeout(() => { + if (finished) { + return + } + + finished = true + tar.kill('SIGTERM') + ssh.kill('SIGTERM') + reject(new Error(`timed out uploading ${params.sourceDir} to ${remoteDir}`)) + }, timeoutMs) + + const cleanup = () => { + clearTimeout(timer) + } + + tar.stderr.on('data', (chunk) => { + tarStderr += chunk.toString('utf-8') + }) + ssh.stderr.on('data', (chunk) => { + sshStderr += chunk.toString('utf-8') + }) + + tar.on('error', (error) => { + if (finished) { + return + } + + finished = true + cleanup() + ssh.kill('SIGTERM') + reject(error) + }) + + ssh.on('error', (error) => { + if (finished) { + return + } + + finished = true + cleanup() + tar.kill('SIGTERM') + reject(error) + }) + + tar.stdout.pipe(ssh.stdin) + + let tarExited = false + let sshExited = false + let tarCode = 0 + let sshCode = 0 + + const maybeResolve = () => { + if (finished || !tarExited || !sshExited) { + return + } + + finished = true + cleanup() + + if (tarCode !== 0) { + reject(new Error(tarStderr.trim() || `tar exited with code ${tarCode}`)) + return + } + + if (sshCode !== 0) { + reject(new Error(sshStderr.trim() || `ssh upload exited with code ${sshCode}`)) + return + } + + resolve() + } + + tar.on('close', (code) => { + tarExited = true + tarCode = code ?? 1 + maybeResolve() + }) + + ssh.on('close', (code) => { + sshExited = true + sshCode = code ?? 1 + maybeResolve() + }) + }) +} diff --git a/services/computer-use-mcp/src/reroute-contract.ts b/services/computer-use-mcp/src/reroute-contract.ts new file mode 100644 index 000000000..fc644ef2e --- /dev/null +++ b/services/computer-use-mcp/src/reroute-contract.ts @@ -0,0 +1,74 @@ +/** + * Stable outward reroute contract. + * + * This is the authoritative shape that `workflow_*` tools emit inside + * `structuredContent` when the workflow engine decides to reroute. + * + * Consumers (stage-ui, external integrations) MUST rely exclusively + * on these types — they are considered a forward-compatible public API + * surface. Field semantics are documented inline and must not change + * meaning across versions. + */ + +import type { RecommendedSurface } from './strategy' +import type { BrowserSurfaceKind, TerminalSurface } from './types' + +// --------------------------------------------------------------------------- +// Reroute detail block +// --------------------------------------------------------------------------- + +export interface WorkflowRerouteDetail { + /** Target surface category for the reroute. */ + recommendedSurface: RecommendedSurface + /** Most-recommended next tool to call. */ + suggestedTool: string + /** Why the strategy layer decided to reroute (always present). */ + strategyReason: string + /** + * Only present when a prep tool or runtime probe ran and provided + * information beyond the strategy heuristic. Pure-strategy reroutes + * MUST NOT fabricate this field. + */ + executionReason?: string + /** Human-readable explanation for logging / UI display. */ + explanation: string + /** + * Available browser surface stacks at the moment of reroute. + * Only populated for browser-family reroutes. + */ + availableSurfaces?: BrowserSurfaceKind[] + /** + * The preferred browser surface stack chosen by the runtime + * availability model. Only populated for browser-family reroutes. + */ + preferredSurface?: BrowserSurfaceKind + /** + * Terminal surface that triggered the reroute (e.g. 'pty'). + * Only populated for terminal-family reroutes (exec → pty). + */ + terminalSurface?: TerminalSurface + /** + * PTY session id to resume (if the reroute is to an existing PTY). + * Only populated for pty reroutes with bound sessions. + */ + ptySessionId?: string +} + +// --------------------------------------------------------------------------- +// Top-level structuredContent shape for reroute responses +// --------------------------------------------------------------------------- + +export interface WorkflowRerouteStructuredContent { + kind: 'workflow_reroute' + status: 'reroute_required' + workflow: string + reroute: WorkflowRerouteDetail + task: unknown + stepResults: unknown[] +} + +// --------------------------------------------------------------------------- +// Builder +// --------------------------------------------------------------------------- + +export type { RecommendedSurface } from './strategy' diff --git a/services/computer-use-mcp/src/runner/client.test.ts b/services/computer-use-mcp/src/runner/client.test.ts new file mode 100644 index 000000000..64adfa827 --- /dev/null +++ b/services/computer-use-mcp/src/runner/client.test.ts @@ -0,0 +1,85 @@ +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { describe, expect, it } from 'vitest' + +import { createTestConfig } from '../test-fixtures' +import { RemoteRunnerClient } from './client' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const fakeRunnerPath = resolve(packageDir, 'fixtures/fake-runner.mjs') + +function createFakeRunnerTransport(env: NodeJS.ProcessEnv = {}) { + return () => ({ + command: process.execPath, + args: [fakeRunnerPath], + env: { + ...process.env, + ...env, + }, + }) +} + +describe('remoteRunnerClient', () => { + it('connects to a runner and returns remote execution metadata', async () => { + const client = new RemoteRunnerClient(createTestConfig(), { + transportFactory: createFakeRunnerTransport({ + FAKE_RUNNER_OBSERVATION_BASE_URL: 'http://fake-remote:8765/observations/vm-local-1', + }), + }) + + try { + const target = await client.getExecutionTarget() + const screenshot = await client.takeScreenshot({ + label: 'client-test', + }) + + expect(target.mode).toBe('remote') + expect(target.transport).toBe('ssh-stdio') + expect(target.sessionTag).toBe('vm-local-1') + expect(target.displayId).toBe(':99') + expect(screenshot.width).toBe(1280) + expect(screenshot.executionTarget.hostName).toBe('fake-remote') + expect(screenshot.publicUrl).toBe('http://fake-remote:8765/observations/vm-local-1/fake-screenshot.png') + } + finally { + await client.close() + } + }) + + it('keeps the remote session tainted until a fresh screenshot succeeds', async () => { + let closeOnMutation = true + const client = new RemoteRunnerClient(createTestConfig(), { + transportFactory: () => ({ + command: process.execPath, + args: [fakeRunnerPath], + env: { + ...process.env, + ...(closeOnMutation ? { FAKE_RUNNER_CLOSE_ON_MUTATION: '1' } : {}), + }, + }), + }) + + try { + await client.getExecutionTarget() + await expect(client.click({ + x: 180, + y: 150, + pointerTrace: [{ x: 180, y: 150, delayMs: 0 }], + })).rejects.toThrow() + + const taintedTarget = await client.getExecutionTarget() + expect(taintedTarget.tainted).toBe(true) + + closeOnMutation = false + await client.takeScreenshot({ + label: 'fresh-screenshot', + }) + const recoveredTarget = await client.getExecutionTarget() + expect(recoveredTarget.tainted).toBe(false) + } + finally { + await client.close().catch(() => {}) + } + }) +}) diff --git a/services/computer-use-mcp/src/runner/client.ts b/services/computer-use-mcp/src/runner/client.ts new file mode 100644 index 000000000..2969950c2 --- /dev/null +++ b/services/computer-use-mcp/src/runner/client.ts @@ -0,0 +1,381 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' + +import type { + ClickActionInput, + ComputerUseConfig, + DisplayInfo, + ExecutionTarget, + ExecutorActionResult, + ForegroundContext, + PermissionInfo, + PointerTracePoint, + PressKeysActionInput, + ScreenshotRequest, + ScrollActionInput, + TypeTextActionInput, + WaitActionInput, +} from '../types' +import type { + RunnerActionResult, + RunnerInitializeResult, + RunnerMethod, + RunnerOpenTestTargetResult, + RunnerRequest, + RunnerRequestParams, + RunnerResponse, + RunnerScreenshotResult, +} from './protocol' + +import { spawn } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { createInterface } from 'node:readline' + +function quoteShell(value: string) { + return `'${value.replaceAll(`'`, `'\\''`)}'` +} + +function buildSshTarget(config: ComputerUseConfig) { + if (!config.remoteSshHost || !config.remoteSshUser) { + throw new Error('linux-x11 executor requires COMPUTER_USE_REMOTE_SSH_HOST and COMPUTER_USE_REMOTE_SSH_USER') + } + + return `${config.remoteSshUser}@${config.remoteSshHost}` +} + +export interface RunnerTransportCommand { + command: string + args: string[] + env?: NodeJS.ProcessEnv +} + +export type RunnerTransportFactory = () => RunnerTransportCommand + +interface PendingRequest { + resolve: (value: unknown) => void + reject: (error: Error) => void +} + +export interface RemoteRunnerClientOptions { + transportFactory?: RunnerTransportFactory +} + +export function createSshRunnerTransportFactory(config: ComputerUseConfig): RunnerTransportFactory { + return () => ({ + command: config.binaries.ssh, + args: [ + '-T', + '-p', + String(config.remoteSshPort), + '-o', + 'BatchMode=yes', + '-o', + 'ServerAliveInterval=30', + '-o', + 'ServerAliveCountMax=3', + buildSshTarget(config), + 'sh', + '-lc', + quoteShell(config.remoteRunnerCommand), + ], + env: process.env, + }) +} + +export class RemoteRunnerClient { + private readonly pending = new Map() + private readonly transportFactory: RunnerTransportFactory + private readonly config: ComputerUseConfig + private readonly requestQueue: Promise = Promise.resolve() + private queueTail = this.requestQueue + private child?: ChildProcessWithoutNullStreams + private currentTarget?: ExecutionTarget + private currentPermissions?: PermissionInfo + private connecting?: Promise + private taintedReason?: string + + constructor(config: ComputerUseConfig, options: RemoteRunnerClientOptions = {}) { + this.config = config + this.transportFactory = options.transportFactory || createSshRunnerTransportFactory(config) + } + + async getExecutionTarget() { + await this.ensureConnected() + return await this.call('getExecutionTarget') + } + + async getDisplayInfo() { + await this.ensureConnected() + return await this.call('getDisplayInfo') + } + + async getForegroundContext() { + await this.ensureConnected() + return await this.call('getForegroundContext') + } + + async getPermissionInfo() { + await this.ensureConnected() + if (!this.currentPermissions) { + this.currentPermissions = await this.call('getPermissionInfo') + } + return this.currentPermissions + } + + async takeScreenshot(request: ScreenshotRequest) { + await this.ensureConnected() + const result = await this.call('takeScreenshot', request) + this.taintedReason = undefined + this.currentTarget = this.applyPersistentTaint(result.executionTarget) + return { + ...result, + executionTarget: this.currentTarget, + } + } + + async click(input: ClickActionInput & { pointerTrace: PointerTracePoint[] }) { + await this.ensureConnected() + return await this.call('click', input, { mutating: true }) + } + + async typeText(input: TypeTextActionInput) { + await this.ensureConnected() + return await this.call('typeText', input, { mutating: true }) + } + + async pressKeys(input: PressKeysActionInput) { + await this.ensureConnected() + return await this.call('pressKeys', input, { mutating: true }) + } + + async scroll(input: ScrollActionInput) { + await this.ensureConnected() + return await this.call('scroll', input, { mutating: true }) + } + + async wait(input: WaitActionInput) { + await this.ensureConnected() + return await this.call('wait', input) + } + + async openTestTarget() { + await this.ensureConnected() + return await this.call('openTestTarget') + } + + async close() { + if (!this.child) + return + + try { + await this.call('shutdown') + } + catch { + // Ignore shutdown errors and kill the transport below. + } + + this.child.kill('SIGTERM') + this.child = undefined + this.currentTarget = undefined + this.currentPermissions = undefined + this.taintedReason = undefined + } + + private async ensureConnected() { + if (this.child) + return + + if (this.connecting) { + await this.connecting + return + } + + this.connecting = this.connect() + try { + await this.connecting + } + finally { + this.connecting = undefined + } + } + + private async connect() { + const transport = this.transportFactory() + const child = spawn(transport.command, transport.args, { + env: transport.env, + stdio: ['pipe', 'pipe', 'pipe'], + }) + + let stderr = '' + child.stderr.on('data', (chunk) => { + stderr += chunk.toString('utf-8') + if (stderr.length > 16_384) { + stderr = stderr.slice(-16_384) + } + }) + + const rl = createInterface({ + input: child.stdout, + crlfDelay: Infinity, + }) + + rl.on('line', (line) => { + const trimmed = line.trim() + if (!trimmed) + return + + let response: RunnerResponse + try { + response = JSON.parse(trimmed) as RunnerResponse + } + catch { + return + } + + const pending = this.pending.get(response.id) + if (!pending) + return + + this.pending.delete(response.id) + if (response.ok === true) { + pending.resolve(response.result) + return + } + + pending.reject(new Error(response.error.message)) + }) + + const markTainted = (reason: string) => { + this.taintedReason = reason + if (this.currentTarget) { + this.currentTarget = this.applyPersistentTaint(this.currentTarget) + } + } + + const rejectPending = (error: Error) => { + for (const pending of this.pending.values()) { + pending.reject(error) + } + this.pending.clear() + } + + child.on('error', (error) => { + markTainted(error.message) + rejectPending(error) + this.child = undefined + }) + + child.on('close', (_code, signal) => { + markTainted(`remote runner transport closed${signal ? ` (${signal})` : ''}`) + rejectPending(new Error(stderr.trim() || 'remote runner transport closed unexpectedly')) + this.child = undefined + }) + + this.child = child + + const initial = await this.rawCall('initialize', { + sessionTag: this.config.sessionTag, + displaySize: this.config.remoteDisplaySize, + observationBaseUrl: this.config.remoteObservationBaseUrl, + observationServePort: this.config.remoteObservationServePort, + observationToken: this.config.remoteObservationToken, + }) + this.currentTarget = this.applyPersistentTaint(initial.executionTarget) + this.currentPermissions = initial.permissionInfo + } + + private async call(method: RunnerMethod, params?: RunnerRequestParams, options: { + mutating?: boolean + } = {}) { + try { + const result = await this.rawCall(method, params) + if (method === 'getExecutionTarget') { + this.currentTarget = this.applyPersistentTaint(result as ExecutionTarget) + return this.currentTarget as Result + } + + const resultWithExecutionTarget = result as { executionTarget?: ExecutionTarget } + if (resultWithExecutionTarget.executionTarget) { + this.currentTarget = this.applyPersistentTaint(resultWithExecutionTarget.executionTarget) + return { + ...(resultWithExecutionTarget as Record), + executionTarget: this.currentTarget, + } as Result + } + + return result + } + catch (error) { + if (options.mutating) { + this.taintedReason = error instanceof Error ? error.message : String(error) + } + + if (this.currentTarget) { + this.currentTarget = this.applyPersistentTaint({ + ...this.currentTarget, + note: error instanceof Error ? error.message : String(error), + }) + } + throw error + } + } + + private async rawCall(method: RunnerMethod, params?: RunnerRequestParams): Promise { + return await this.enqueue(async () => { + if (!this.child) { + throw new Error(`remote runner is not connected for ${method}`) + } + + const request: RunnerRequest = { + id: randomUUID(), + method, + params, + } + + const result = await new Promise((resolve, reject) => { + this.pending.set(request.id, { resolve: resolve as (value: unknown) => void, reject }) + this.child?.stdin.write(`${JSON.stringify(request)}\n`, 'utf-8', (error) => { + if (!error) + return + + this.pending.delete(request.id) + reject(error) + }) + }) + + return result + }) + } + + private async enqueue(task: () => Promise) { + const next = this.queueTail.then(task, task) + this.queueTail = next.then(() => undefined, () => undefined) + return await next + } + + private applyPersistentTaint(target: ExecutionTarget): ExecutionTarget { + if (!this.taintedReason || target.mode !== 'remote') { + return { + ...target, + tainted: target.tainted, + } + } + + return { + ...target, + tainted: true, + note: this.taintedReason, + } + } +} + +export interface CreateLocalLinuxExecutorActionResult { + click: RunnerActionResult + typeText: RunnerActionResult + pressKeys: RunnerActionResult + scroll: RunnerActionResult + wait: RunnerActionResult +} + +export function toExecutorActionResult(result: RunnerActionResult): ExecutorActionResult { + return result +} diff --git a/services/computer-use-mcp/src/runner/protocol.ts b/services/computer-use-mcp/src/runner/protocol.ts new file mode 100644 index 000000000..91b223404 --- /dev/null +++ b/services/computer-use-mcp/src/runner/protocol.ts @@ -0,0 +1,93 @@ +import type { + ClickActionInput, + DisplayInfo, + DisplaySize, + ExecutionTarget, + ExecutorActionResult, + PermissionInfo, + PointerTracePoint, + PressKeysActionInput, + ScreenshotRequest, + ScrollActionInput, + TestTargetLaunchResult, + TypeTextActionInput, + WaitActionInput, +} from '../types' + +export type RunnerMethod + = | 'initialize' + | 'getExecutionTarget' + | 'getDisplayInfo' + | 'getForegroundContext' + | 'getPermissionInfo' + | 'takeScreenshot' + | 'click' + | 'typeText' + | 'pressKeys' + | 'scroll' + | 'wait' + | 'openTestTarget' + | 'shutdown' + +export interface RunnerInitializeParams { + sessionTag?: string + displaySize: DisplaySize + observationBaseUrl?: string + observationServePort?: number + observationToken?: string +} + +export interface RunnerInitializeResult { + executionTarget: ExecutionTarget + displayInfo: DisplayInfo + permissionInfo: PermissionInfo +} + +export interface RunnerScreenshotResult { + dataBase64: string + mimeType: 'image/png' + publicUrl?: string + width?: number + height?: number + note?: string + executionTarget: ExecutionTarget +} + +export type RunnerOpenTestTargetResult = TestTargetLaunchResult + +export interface RunnerActionResult extends ExecutorActionResult { + executionTarget: ExecutionTarget +} + +export type RunnerRequestParams + = | RunnerInitializeParams + | ScreenshotRequest + | (ClickActionInput & { pointerTrace: PointerTracePoint[] }) + | TypeTextActionInput + | PressKeysActionInput + | ScrollActionInput + | WaitActionInput + | Record + +export interface RunnerRequest { + id: string + method: RunnerMethod + params?: RunnerRequestParams +} + +export interface RunnerSuccessResponse { + id: string + ok: true + result: Result +} + +export interface RunnerErrorResponse { + id: string + ok: false + error: { + message: string + code?: string + } +} + +export type RunnerResponse = RunnerSuccessResponse | RunnerErrorResponse diff --git a/services/computer-use-mcp/src/runner/service.ts b/services/computer-use-mcp/src/runner/service.ts new file mode 100644 index 000000000..8ad43999c --- /dev/null +++ b/services/computer-use-mcp/src/runner/service.ts @@ -0,0 +1,682 @@ +import type { Server } from 'node:http' + +import type { + ClickActionInput, + DisplayInfo, + DisplaySize, + ExecutionTarget, + ForegroundContext, + PermissionInfo, + PointerTracePoint, + PressKeysActionInput, + ScrollActionInput, + TypeTextActionInput, + WaitActionInput, +} from '../types' +import type { + RunnerActionResult, + RunnerInitializeParams, + RunnerInitializeResult, + RunnerOpenTestTargetResult, + RunnerScreenshotResult, +} from './protocol' + +import { spawn } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { createReadStream } from 'node:fs' +import { access, mkdir, mkdtemp, readFile, rm } from 'node:fs/promises' +import { createServer } from 'node:http' +import { homedir, tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { platform } from 'node:process' + +import { runProcess, sanitizeFileSegment } from '../utils/process' + +const sessionDisplayStart = 90 +const sessionDisplayEnd = 110 + +async function sleep(durationMs: number) { + await new Promise(resolve => setTimeout(resolve, durationMs)) +} + +function toExecutionTarget(params: { + hostName: string + displayId?: string + sessionTag?: string + tainted?: boolean + note?: string +}): ExecutionTarget { + return { + mode: 'remote', + transport: 'ssh-stdio', + hostName: params.hostName, + displayId: params.displayId, + sessionTag: params.sessionTag, + isolated: true, + tainted: params.tainted ?? false, + note: params.note, + } +} + +function mapButton(button: ClickActionInput['button']) { + switch (button) { + case 'right': + return '3' + case 'middle': + return '2' + default: + return '1' + } +} + +function normalizeKey(key: string) { + switch (key.trim().toLowerCase()) { + case 'cmd': + case 'command': + return 'Super_L' + case 'ctrl': + case 'control': + return 'Control_L' + case 'alt': + case 'option': + return 'Alt_L' + case 'shift': + return 'Shift_L' + case 'enter': + return 'Return' + case 'esc': + return 'Escape' + case 'space': + return 'space' + default: + return key.trim() + } +} + +export class LinuxX11RunnerService { + private initialized = false + private displayId?: string + private observationBaseUrl?: URL + private observationPublicDir?: string + private observationServer?: Server + private observationServePort?: number + private observationToken?: string + private xAuthorityPath?: string + private runtimeDir?: string + private openboxPid?: number + private xvfbPid?: number + private sessionTag?: string + private displaySize?: DisplaySize + private target: ExecutionTarget = toExecutionTarget({ + hostName: platform === 'linux' ? 'unknown-linux-runner' : 'unknown-runner', + tainted: false, + }) + + async initialize(params: RunnerInitializeParams): Promise { + if (platform !== 'linux') { + throw new Error(`linux-x11 runner only supports linux hosts, current platform is ${platform}`) + } + + if (this.initialized) { + if (params.sessionTag !== this.sessionTag) { + throw new Error(`runner already initialized for session ${this.sessionTag || 'unknown'}`) + } + + return { + executionTarget: this.requireExecutionTarget(), + displayInfo: await this.getDisplayInfo(), + permissionInfo: await this.getPermissionInfo(), + } + } + + await this.ensureDependencies() + + this.sessionTag = params.sessionTag + this.displaySize = params.displaySize + this.runtimeDir = await mkdtemp(join(tmpdir(), `airi-linux-x11-${sanitizeFileSegment(params.sessionTag, 'session')}-`)) + this.observationBaseUrl = params.observationBaseUrl ? new URL(params.observationBaseUrl) : undefined + this.observationServePort = params.observationServePort + this.observationToken = params.observationToken?.trim() || randomBytes(12).toString('hex') + this.observationPublicDir = join(this.runtimeDir, 'published-observations') + this.xAuthorityPath = join(this.runtimeDir, 'Xauthority') + this.displayId = await this.allocateDisplayId() + + await this.initializeXAuthority() + await this.startXvfb() + await this.waitForDisplay() + await this.startOpenbox() + await this.startObservationServer() + + this.target = toExecutionTarget({ + hostName: await this.getHostName(), + displayId: this.displayId, + sessionTag: this.sessionTag, + tainted: false, + }) + this.initialized = true + + return { + executionTarget: this.requireExecutionTarget(), + displayInfo: await this.getDisplayInfo(), + permissionInfo: await this.getPermissionInfo(), + } + } + + async shutdown() { + await this.stopObservationServer() + await this.killProcess(this.openboxPid) + await this.killProcess(this.xvfbPid) + if (this.runtimeDir) { + await rm(this.runtimeDir, { recursive: true, force: true }) + } + this.initialized = false + } + + async getExecutionTarget() { + return this.requireExecutionTarget() + } + + async getDisplayInfo(): Promise { + await this.ensureReady() + return { + available: true, + platform: 'linux', + logicalWidth: this.displaySize?.width, + logicalHeight: this.displaySize?.height, + pixelWidth: this.displaySize?.width, + pixelHeight: this.displaySize?.height, + scaleFactor: 1, + isRetina: false, + note: `managed virtual X session ${this.displayId}`, + } + } + + async getForegroundContext(): Promise { + await this.ensureReady() + + try { + const { stdout } = await runProcess('xprop', ['-root', '_NET_ACTIVE_WINDOW'], { + timeoutMs: 5_000, + env: this.getX11Env(), + }) + const match = stdout.match(/window id # (0x[0-9a-fA-F]+)/) + if (!match || match[1] === '0x0') { + return { + available: false, + platform: 'linux', + unavailableReason: 'no active window in managed X session', + } + } + + const windowId = match[1] + const [titleResult, classResult] = await Promise.all([ + runProcess('xprop', ['-id', windowId, '_NET_WM_NAME'], { + timeoutMs: 5_000, + env: this.getX11Env(), + }).catch(() => ({ stdout: '', stderr: '' })), + runProcess('xprop', ['-id', windowId, 'WM_CLASS'], { + timeoutMs: 5_000, + env: this.getX11Env(), + }).catch(() => ({ stdout: '', stderr: '' })), + ]) + + const title = titleResult.stdout.match(/=\s*"([^"]*)"/)?.[1] + const classes = classResult.stdout.match(/=\s*"([^"]*)",\s*"([^"]*)"/) + + return { + available: true, + appName: classes?.[2] || classes?.[1] || undefined, + windowTitle: title || undefined, + platform: 'linux', + } + } + catch (error) { + return { + available: false, + platform: 'linux', + unavailableReason: error instanceof Error ? error.message : String(error), + } + } + } + + async getPermissionInfo(): Promise { + await this.ensureReady() + + return { + screenRecording: { + status: 'granted', + target: `${this.displayId} via scrot`, + checkedBy: 'scrot', + }, + accessibility: { + status: 'unsupported', + target: `${this.displayId} linux-x11 session`, + note: 'linux-x11 runner does not rely on accessibility APIs', + }, + automationToSystemEvents: { + status: 'unsupported', + target: `${this.displayId} linux-x11 session`, + note: 'linux-x11 runner does not use System Events', + }, + } + } + + async takeScreenshot(params: { label?: string }): Promise { + await this.ensureReady() + + const fileName = this.observationBaseUrl + ? `${Date.now()}-${randomBytes(8).toString('hex')}-${sanitizeFileSegment(params.label, 'desktop')}.png` + : `${Date.now()}-${sanitizeFileSegment(params.label, 'desktop')}.png` + const outputPath = join(this.observationPublicDir || this.runtimeDir!, fileName) + await runProcess('scrot', ['-z', '-q', '100', outputPath], { + timeoutMs: 10_000, + env: this.getX11Env(), + }) + + const buffer = await readFile(outputPath) + if (!this.observationBaseUrl) { + await rm(outputPath, { force: true }) + } + + return { + dataBase64: buffer.toString('base64'), + mimeType: 'image/png', + publicUrl: this.buildObservationPublicUrl(fileName), + width: this.displaySize?.width, + height: this.displaySize?.height, + executionTarget: this.requireExecutionTarget(), + } + } + + async click(input: ClickActionInput & { pointerTrace: PointerTracePoint[] }): Promise { + await this.ensureReady() + + for (const point of input.pointerTrace) { + await this.movePointer(point.x, point.y) + await sleep(point.delayMs) + } + + await runProcess('xdotool', ['click', '--repeat', String(input.clickCount ?? 1), mapButton(input.button)], { + timeoutMs: 5_000, + env: this.getX11Env(), + }) + + return { + performed: true, + backend: 'linux-x11', + notes: [`clicked ${input.x},${input.y} in ${this.displayId}`], + pointerTrace: input.pointerTrace, + executionTarget: this.requireExecutionTarget(), + } + } + + async typeText(input: TypeTextActionInput): Promise { + await this.ensureReady() + + await runProcess('xdotool', ['type', '--delay', '15', '--clearmodifiers', '--', input.text], { + timeoutMs: 10_000, + env: this.getX11Env(), + }) + if (input.pressEnter) { + await runProcess('xdotool', ['key', '--clearmodifiers', 'Return'], { + timeoutMs: 5_000, + env: this.getX11Env(), + }) + } + + return { + performed: true, + backend: 'linux-x11', + notes: ['typed text in managed X session'], + executionTarget: this.requireExecutionTarget(), + } + } + + async pressKeys(input: PressKeysActionInput): Promise { + await this.ensureReady() + + const chord = input.keys.map(normalizeKey).join('+') + await runProcess('xdotool', ['key', '--clearmodifiers', chord], { + timeoutMs: 5_000, + env: this.getX11Env(), + }) + + return { + performed: true, + backend: 'linux-x11', + notes: [`pressed key chord ${chord}`], + executionTarget: this.requireExecutionTarget(), + } + } + + async scroll(input: ScrollActionInput): Promise { + await this.ensureReady() + + if (typeof input.x === 'number' && typeof input.y === 'number') { + await this.movePointer(input.x, input.y) + } + + const verticalSteps = Math.max(1, Math.ceil(Math.abs(input.deltaY) / 120)) + const verticalButton = input.deltaY < 0 ? '4' : '5' + for (let index = 0; index < verticalSteps; index += 1) { + await runProcess('xdotool', ['click', verticalButton], { + timeoutMs: 5_000, + env: this.getX11Env(), + }) + } + + if (input.deltaX) { + const horizontalSteps = Math.max(1, Math.ceil(Math.abs(input.deltaX) / 120)) + const horizontalButton = input.deltaX < 0 ? '6' : '7' + for (let index = 0; index < horizontalSteps; index += 1) { + await runProcess('xdotool', ['click', horizontalButton], { + timeoutMs: 5_000, + env: this.getX11Env(), + }) + } + } + + return { + performed: true, + backend: 'linux-x11', + notes: ['scrolled in managed X session'], + executionTarget: this.requireExecutionTarget(), + } + } + + async wait(input: WaitActionInput): Promise { + await this.ensureReady() + await sleep(Math.max(input.durationMs, 0)) + + return { + performed: true, + backend: 'linux-x11', + notes: ['waited in managed X session'], + executionTarget: this.requireExecutionTarget(), + } + } + + async openTestTarget(): Promise { + await this.ensureReady() + + const child = spawn('mousepad', ['--disable-server'], { + env: this.getX11Env(), + stdio: 'ignore', + detached: false, + }) + + const windowId = await this.waitForWindow(child.pid) + await runProcess('wmctrl', ['-i', '-r', windowId, '-e', '0,80,40,1000,620'], { + timeoutMs: 5_000, + env: this.getX11Env(), + }).catch(() => {}) + await runProcess('wmctrl', ['-i', '-a', windowId], { + timeoutMs: 5_000, + env: this.getX11Env(), + }).catch(() => {}) + // NOTICE: xdotool --sync can hang under Xvfb/openbox when the target window + // is already focused or a pointer move is effectively a no-op. Keep activation + // best-effort and rely on a short settle delay instead of sync waits. + await runProcess('xdotool', ['windowactivate', windowId], { + timeoutMs: 5_000, + env: this.getX11Env(), + }).catch(() => {}) + await sleep(100) + + return { + launched: true, + appName: 'mousepad', + windowTitle: 'Mousepad', + recommendedClickPoint: { + x: 180, + y: 150, + }, + executionTarget: this.requireExecutionTarget(), + } + } + + private async ensureDependencies() { + for (const binary of ['Xvfb', 'xauth', 'xdotool', 'wmctrl', 'scrot', 'openbox', 'xdpyinfo', 'xprop']) { + await runProcess('which', [binary], { + timeoutMs: 5_000, + }).catch(() => { + throw new Error(`missing required linux-x11 runner dependency: ${binary}`) + }) + } + } + + private async allocateDisplayId() { + for (let displayNumber = sessionDisplayStart; displayNumber <= sessionDisplayEnd; displayNumber += 1) { + const displayId = `:${displayNumber}` + const available = await runProcess('xdpyinfo', ['-display', displayId], { + timeoutMs: 1_500, + env: this.getX11Env(displayId), + }).then(() => false).catch(() => true) + + if (available) + return displayId + } + + throw new Error(`unable to allocate a free X display between :${sessionDisplayStart} and :${sessionDisplayEnd}`) + } + + private async initializeXAuthority() { + const cookie = randomBytes(16).toString('hex') + await runProcess('xauth', ['-f', this.xAuthorityPath!, 'add', this.displayId!, '.', cookie], { + timeoutMs: 5_000, + }) + } + + private async startXvfb() { + const child = spawn('Xvfb', [ + this.displayId!, + '-screen', + '0', + `${this.displaySize!.width}x${this.displaySize!.height}x24`, + '-nolisten', + 'tcp', + '-auth', + this.xAuthorityPath!, + ], { + env: this.getX11Env(), + stdio: 'ignore', + }) + + this.xvfbPid = child.pid + } + + private async startOpenbox() { + const child = spawn('openbox', [], { + env: this.getX11Env(), + stdio: 'ignore', + }) + + this.openboxPid = child.pid + } + + private async waitForDisplay() { + for (let attempt = 0; attempt < 30; attempt += 1) { + const ready = await runProcess('xdpyinfo', ['-display', this.displayId!], { + timeoutMs: 1_500, + env: this.getX11Env(), + }).then(() => true).catch(() => false) + + if (ready) + return + + await sleep(200) + } + + throw new Error(`timed out waiting for virtual display ${this.displayId}`) + } + + private async waitForWindow(pid?: number) { + if (!pid) { + throw new Error('test target did not provide a process id') + } + + for (let attempt = 0; attempt < 40; attempt += 1) { + const { stdout } = await runProcess('wmctrl', ['-lp'], { + timeoutMs: 5_000, + env: this.getX11Env(), + }).catch(() => ({ stdout: '', stderr: '' })) + + const match = stdout.split(/\r?\n/).find((line) => { + return line.trim().split(/\s+/)[2] === String(pid) + }) + + if (match) { + return match.trim().split(/\s+/)[0] + } + + await sleep(250) + } + + throw new Error('timed out waiting for the mousepad window') + } + + private async killProcess(pid?: number) { + if (!pid) + return + + await runProcess('kill', ['-TERM', String(pid)], { + timeoutMs: 5_000, + }).catch(() => {}) + } + + private async getHostName() { + const { stdout } = await runProcess('hostname', [], { + timeoutMs: 5_000, + }) + return stdout.trim() || homedir() + } + + private requireExecutionTarget() { + if (!this.initialized || !this.displayId || !this.displaySize) { + throw new Error('linux-x11 runner is not initialized') + } + + return { + ...this.target, + hostName: this.target.hostName, + displayId: this.displayId, + sessionTag: this.sessionTag, + } + } + + private async ensureReady() { + if (!this.initialized) { + throw new Error('linux-x11 runner is not initialized') + } + } + + private async startObservationServer() { + if (!this.observationBaseUrl || !this.observationServePort || !this.observationPublicDir) { + return + } + + await mkdir(this.observationPublicDir, { recursive: true }) + + const basePath = this.getObservationBasePath() + const routePrefix = `${basePath}/${this.observationToken}`.replace(/\/{2,}/g, '/') + + this.observationServer = createServer(async (request, response) => { + const pathname = (request.url || '/').split('?')[0] || '/' + if (!pathname.startsWith(`${routePrefix}/`)) { + response.writeHead(404) + response.end('not found') + return + } + + const requestedName = basename(pathname.slice(routePrefix.length + 1)) + if (!requestedName.endsWith('.png')) { + response.writeHead(404) + response.end('not found') + return + } + + const filePath = join(this.observationPublicDir!, requestedName) + try { + await access(filePath) + } + catch { + response.writeHead(404) + response.end('not found') + return + } + + const stream = createReadStream(filePath) + stream.on('error', () => { + if (!response.headersSent) { + response.writeHead(404) + response.end('not found') + return + } + + response.destroy() + }) + + response.writeHead(200, { + 'Cache-Control': 'no-store', + 'Content-Type': 'image/png', + }) + stream.pipe(response) + }) + + await new Promise((resolve, reject) => { + this.observationServer!.once('error', reject) + this.observationServer!.listen(this.observationServePort, '0.0.0.0', () => { + this.observationServer?.off('error', reject) + resolve() + }) + }) + } + + private async stopObservationServer() { + if (!this.observationServer) { + return + } + + const server = this.observationServer + this.observationServer = undefined + await new Promise((resolve) => { + server.close(() => resolve()) + }) + } + + private getObservationBasePath() { + if (!this.observationBaseUrl) { + return '' + } + + return this.observationBaseUrl.pathname.replace(/\/$/, '') + } + + private buildObservationPublicUrl(fileName: string) { + if (!this.observationBaseUrl || !this.observationToken) { + return undefined + } + + const basePath = this.getObservationBasePath() + const pathName = `${basePath}/${this.observationToken}/${fileName}`.replace(/\/{2,}/g, '/') + return new URL(pathName, this.observationBaseUrl).toString() + } + + private getX11Env(displayOverride?: string) { + return { + ...process.env, + DISPLAY: displayOverride || this.displayId, + XAUTHORITY: this.xAuthorityPath, + } + } + + private async movePointer(x: number, y: number) { + // NOTICE: repeated clicks at the same coordinate are a normal computer-use flow. + // `xdotool mousemove --sync` waits for an actual pointer movement and can block + // forever when the pointer is already at the requested position under Xvfb/openbox. + await runProcess('xdotool', ['mousemove', String(x), String(y)], { + timeoutMs: 5_000, + env: this.getX11Env(), + }) + } +} diff --git a/services/computer-use-mcp/src/runtime-probes.test.ts b/services/computer-use-mcp/src/runtime-probes.test.ts new file mode 100644 index 000000000..01d6a9378 --- /dev/null +++ b/services/computer-use-mcp/src/runtime-probes.test.ts @@ -0,0 +1,62 @@ +import type { ComputerUseConfig } from './types' + +import { describe, expect, it } from 'vitest' + +import { buildCoordinateSpaceInfo } from './runtime-probes' +import { createTestConfig } from './test-fixtures' + +const baseConfig: ComputerUseConfig = createTestConfig({ + allowedBounds: { x: 0, y: 0, width: 1440, height: 900 }, +}) + +describe('buildCoordinateSpaceInfo', () => { + it('requires a screenshot before real input', () => { + const info = buildCoordinateSpaceInfo({ + config: baseConfig, + }) + + expect(info.readyForMutations).toBe(false) + expect(info.reason).toContain('capture a screenshot') + }) + + it('accepts matching bounds and screenshot dimensions', () => { + const info = buildCoordinateSpaceInfo({ + config: baseConfig, + lastScreenshot: { + path: '/tmp/screenshot.png', + width: 1440, + height: 900, + placeholder: false, + }, + }) + + expect(info.readyForMutations).toBe(true) + expect(info.aligned).toBe(true) + }) + + it('flags logical-vs-physical mismatch on Retina displays', () => { + const info = buildCoordinateSpaceInfo({ + config: baseConfig, + lastScreenshot: { + path: '/tmp/screenshot.png', + width: 2880, + height: 1800, + placeholder: false, + }, + displayInfo: { + available: true, + platform: 'darwin', + logicalWidth: 1440, + logicalHeight: 900, + pixelWidth: 2880, + pixelHeight: 1800, + scaleFactor: 2, + isRetina: true, + }, + }) + + expect(info.readyForMutations).toBe(false) + expect(info.aligned).toBe(false) + expect(info.reason).toContain('Retina') + }) +}) diff --git a/services/computer-use-mcp/src/runtime-probes.ts b/services/computer-use-mcp/src/runtime-probes.ts new file mode 100644 index 000000000..ab42fd855 --- /dev/null +++ b/services/computer-use-mcp/src/runtime-probes.ts @@ -0,0 +1,268 @@ +import type { + ComputerUseConfig, + CoordinateSpaceInfo, + DisplayInfo, + LastScreenshotInfo, + LaunchContext, + PermissionInfo, + PermissionProbe, +} from './types' + +import { hostname } from 'node:os' +import { basename } from 'node:path' +import { argv, pid, platform, ppid, title } from 'node:process' + +import { runProcess } from './utils/process' +import { runSwiftScript } from './utils/swift' + +function unsupportedProbe(target: string, note: string): PermissionProbe { + return { + status: 'unsupported', + target, + note, + } +} + +function inferLaunchHostProcess(config: ComputerUseConfig) { + const argv0 = argv[0]?.trim() + return config.launchHostProcess || basename(argv0 || 'node') +} + +export function resolveLaunchContext(config: ComputerUseConfig): LaunchContext { + const launchHostProcess = inferLaunchHostProcess(config) + + return { + hostName: hostname(), + sessionTag: config.sessionTag, + pid, + ppid, + processTitle: title, + argv: [...argv], + launchHostProcess, + permissionChainHint: config.permissionChainHint || `${launchHostProcess} -> ${config.binaries.osascript} -> System Events`, + } +} + +export async function probeDisplayInfo(config: ComputerUseConfig): Promise { + if (platform !== 'darwin') { + return { + available: false, + platform, + note: 'display probe is only implemented for macOS in this PoC', + } + } + + const script = ` +import AppKit +import Foundation + +guard let screen = NSScreen.main else { + print("{\\"available\\":false,\\"note\\":\\"NSScreen.main unavailable\\"}") + exit(0) +} + +let frame = screen.frame +let scale = screen.backingScaleFactor +let payload: [String: Any] = [ + "available": true, + "logicalWidth": Int(frame.width), + "logicalHeight": Int(frame.height), + "pixelWidth": Int(frame.width * scale), + "pixelHeight": Int(frame.height * scale), + "scaleFactor": scale, + "isRetina": scale > 1.0 +] + +let data = try JSONSerialization.data(withJSONObject: payload, options: []) +print(String(data: data, encoding: .utf8)!) +` + + try { + const { stdout } = await runSwiftScript({ + swiftBinary: config.binaries.swift, + timeoutMs: config.timeoutMs, + source: script, + }) + const parsed = JSON.parse(stdout.trim()) as Omit + + return { + platform, + ...parsed, + } + } + catch (error) { + return { + available: false, + platform, + note: error instanceof Error ? error.message : String(error), + } + } +} + +async function probeAccessibility(config: ComputerUseConfig): Promise { + if (platform !== 'darwin') { + return unsupportedProbe(resolveLaunchContext(config).launchHostProcess, 'accessibility probe is only implemented on macOS') + } + + const script = ` +import ApplicationServices +print(AXIsProcessTrusted() ? "granted" : "missing") +` + + try { + const { stdout } = await runSwiftScript({ + swiftBinary: config.binaries.swift, + timeoutMs: config.timeoutMs, + source: script, + }) + + return { + status: stdout.trim() === 'granted' ? 'granted' : 'missing', + target: resolveLaunchContext(config).launchHostProcess, + checkedBy: 'AXIsProcessTrusted', + } + } + catch (error) { + return { + status: 'unknown', + target: resolveLaunchContext(config).launchHostProcess, + checkedBy: 'AXIsProcessTrusted', + note: error instanceof Error ? error.message : String(error), + } + } +} + +async function probeScreenRecording(config: ComputerUseConfig): Promise { + if (platform !== 'darwin') { + return unsupportedProbe(resolveLaunchContext(config).launchHostProcess, 'screen recording probe is only implemented on macOS') + } + + const script = ` +import CoreGraphics +import Foundation +print(CGPreflightScreenCaptureAccess() ? "granted" : "missing") +` + + try { + const { stdout } = await runSwiftScript({ + swiftBinary: config.binaries.swift, + timeoutMs: config.timeoutMs, + source: script, + }) + + return { + status: stdout.trim() === 'granted' ? 'granted' : 'missing', + target: resolveLaunchContext(config).launchHostProcess, + checkedBy: 'CGPreflightScreenCaptureAccess', + } + } + catch (error) { + return { + status: 'unknown', + target: resolveLaunchContext(config).launchHostProcess, + checkedBy: 'CGPreflightScreenCaptureAccess', + note: error instanceof Error ? error.message : String(error), + } + } +} + +async function probeAutomation(config: ComputerUseConfig): Promise { + const launchContext = resolveLaunchContext(config) + + if (platform !== 'darwin') { + return unsupportedProbe(`${launchContext.launchHostProcess} -> System Events`, 'automation probe is only implemented on macOS') + } + + try { + await runProcess(config.binaries.osascript, [ + '-e', + 'tell application "System Events"', + '-e', + 'return name of first application process whose frontmost is true', + '-e', + 'end tell', + ], { + timeoutMs: config.timeoutMs, + }) + + return { + status: 'granted', + target: `${launchContext.launchHostProcess} -> System Events`, + checkedBy: 'osascript/System Events foreground probe', + } + } + catch (error) { + return { + status: 'missing', + target: `${launchContext.launchHostProcess} -> System Events`, + checkedBy: 'osascript/System Events foreground probe', + note: error instanceof Error ? error.message : String(error), + } + } +} + +export async function probePermissionInfo(config: ComputerUseConfig): Promise { + const [screenRecording, accessibility, automationToSystemEvents] = await Promise.all([ + probeScreenRecording(config), + probeAccessibility(config), + probeAutomation(config), + ]) + + return { + screenRecording, + accessibility, + automationToSystemEvents, + } +} + +export function buildCoordinateSpaceInfo(params: { + config: ComputerUseConfig + lastScreenshot?: LastScreenshotInfo + displayInfo?: DisplayInfo +}): CoordinateSpaceInfo { + const { allowedBounds } = params.config + + if (!allowedBounds) { + return { + readyForMutations: false, + reason: 'allowed bounds are not configured', + allowedBounds, + lastScreenshot: params.lastScreenshot, + } + } + + if (!params.lastScreenshot?.width || !params.lastScreenshot?.height) { + return { + readyForMutations: false, + reason: 'capture a screenshot before real input so the coordinate spaces can be compared', + allowedBounds, + lastScreenshot: params.lastScreenshot, + } + } + + if (allowedBounds.width === params.lastScreenshot.width && allowedBounds.height === params.lastScreenshot.height) { + return { + readyForMutations: true, + aligned: true, + reason: 'screenshot dimensions match allowed bounds', + allowedBounds, + lastScreenshot: params.lastScreenshot, + } + } + + const physicalPixelMismatch = params.displayInfo?.available + && params.displayInfo.pixelWidth === params.lastScreenshot.width + && params.displayInfo.pixelHeight === params.lastScreenshot.height + && params.displayInfo.logicalWidth === allowedBounds.width + && params.displayInfo.logicalHeight === allowedBounds.height + + return { + readyForMutations: false, + aligned: false, + reason: physicalPixelMismatch + ? 'screenshot dimensions match physical pixels while allowed bounds match logical points; align Retina/backing scale before real input' + : `screenshot ${params.lastScreenshot.width}x${params.lastScreenshot.height} does not match allowed bounds ${allowedBounds.width}x${allowedBounds.height}`, + allowedBounds, + lastScreenshot: params.lastScreenshot, + } +} diff --git a/services/computer-use-mcp/src/server.ts b/services/computer-use-mcp/src/server.ts new file mode 100644 index 000000000..8ad46f6bc --- /dev/null +++ b/services/computer-use-mcp/src/server.ts @@ -0,0 +1,81 @@ +import type { ComputerUseServerOptions } from './server/runtime' + +import process, { env } from 'node:process' + +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js' + +import { resolveComputerUseConfig } from './config' +import { createExecuteAction } from './server/action-executor' +import { registerAccessibilityTools } from './server/register-accessibility' +import { registerCdpTools } from './server/register-cdp' +import { registerDisplayTools } from './server/register-display' +import { destroyAllPtySessions, registerPtyTools } from './server/register-pty' +import { registerTaskMemoryTools } from './server/register-task-memory' +import { registerComputerUseTools } from './server/register-tools' +import { registerVscodeTools } from './server/register-vscode' +import { createRuntime } from './server/runtime' + +const packageVersion = '0.1.0' +const enableTestTools = ['1', 'true', 'yes', 'on'].includes((env.COMPUTER_USE_ENABLE_TEST_TOOLS || '').trim().toLowerCase()) + +export { type ComputerUseServerOptions } from './server/runtime' + +export async function createComputerUseMcpServer(config = resolveComputerUseConfig(), options: ComputerUseServerOptions = {}) { + const runtime = await createRuntime(config, options) + const executeAction = createExecuteAction(runtime) + const server = new McpServer({ + name: 'AIRI Computer Use', + version: packageVersion, + }) + + registerComputerUseTools({ + server, + runtime, + executeAction, + enableTestTools, + }) + + registerTaskMemoryTools(server, runtime) + + registerAccessibilityTools({ server, runtime }) + registerDisplayTools({ server, runtime }) + registerPtyTools({ server, runtime }) + registerVscodeTools({ + server, + runtime, + executeTerminalCommand: async (input, toolName) => { + return await executeAction({ kind: 'terminal_exec', input }, toolName) + }, + }) + const cdpCleanup = registerCdpTools({ server, runtime }) + + return { + server, + runtime, + cdpCleanup, + } +} + +export async function startComputerUseMcpServer(config = resolveComputerUseConfig(), options: ComputerUseServerOptions = {}) { + const { server, runtime, cdpCleanup } = await createComputerUseMcpServer(config, options) + const transport = new StdioServerTransport() + await server.connect(transport) + + const shutdown = async () => { + destroyAllPtySessions() + await cdpCleanup.close().catch(() => {}) + await runtime.cdpBridgeManager.close().catch(() => {}) + await runtime.browserDomBridge.close().catch(() => {}) + await runtime.executor.close?.().catch(() => {}) + } + + process.once('SIGINT', () => { + void shutdown() + }) + process.once('SIGTERM', () => { + void shutdown() + }) + + return { server, transport, runtime } +} diff --git a/services/computer-use-mcp/src/server/action-executor.test.ts b/services/computer-use-mcp/src/server/action-executor.test.ts new file mode 100644 index 000000000..51962689a --- /dev/null +++ b/services/computer-use-mcp/src/server/action-executor.test.ts @@ -0,0 +1,110 @@ +import type { ComputerUseServerRuntime } from './runtime' + +import { describe, expect, it, vi } from 'vitest' + +import { RunStateManager } from '../state' +import { createDisplayInfo, createLocalExecutionTarget, createTerminalState, createTestConfig } from '../test-fixtures' +import { createExecuteAction } from './action-executor' + +describe('createExecuteAction', () => { + it('refreshes browser surface availability for direct actions before evaluating strategy', async () => { + const stateManager = new RunStateManager() + const session = { + listPendingActions: vi.fn().mockReturnValue([]), + getBudgetState: vi.fn().mockReturnValue({ + operationsExecuted: 0, + operationUnitsConsumed: 0, + }), + record: vi.fn().mockResolvedValue(undefined), + createPendingAction: vi.fn(), + consumeOperation: vi.fn(), + getLastScreenshot: vi.fn().mockReturnValue(undefined), + setLastScreenshot: vi.fn(), + getTerminalState: vi.fn().mockReturnValue(createTerminalState()), + setTerminalState: vi.fn(), + getPointerPosition: vi.fn().mockReturnValue(undefined), + setPointerPosition: vi.fn(), + } + const executor = { + kind: 'dry-run' as const, + describe: () => ({ kind: 'dry-run' as const, notes: [] }), + getExecutionTarget: vi.fn().mockResolvedValue(createLocalExecutionTarget()), + getForegroundContext: vi.fn().mockResolvedValue({ + available: true, + appName: 'Google Chrome', + platform: 'darwin', + }), + getDisplayInfo: vi.fn().mockResolvedValue(createDisplayInfo({ + platform: 'darwin', + })), + getPermissionInfo: vi.fn(), + observeWindows: vi.fn(), + takeScreenshot: vi.fn(), + openApp: vi.fn(), + focusApp: vi.fn(), + click: vi.fn().mockResolvedValue({ + performed: true, + backend: 'dry-run' as const, + notes: [], + }), + typeText: vi.fn(), + pressKeys: vi.fn(), + scroll: vi.fn(), + wait: vi.fn(), + } + const terminalRunner = { + describe: () => ({ kind: 'local-shell-runner' as const, notes: [] }), + execute: vi.fn(), + getState: vi.fn().mockReturnValue(createTerminalState()), + resetState: vi.fn(), + } + const browserDomBridge = { + getStatus: vi.fn().mockReturnValue({ + enabled: true, + host: '127.0.0.1', + port: 8765, + connected: true, + pendingRequests: 0, + }), + } + const cdpBridgeManager = { + probeAvailability: vi.fn().mockResolvedValue({ + endpoint: 'http://localhost:9222', + connected: false, + connectable: true, + }), + } + + const runtime = { + config: createTestConfig({ + executor: 'dry-run', + approvalMode: 'never', + defaultCaptureAfter: false, + }), + session, + executor, + terminalRunner, + browserDomBridge, + cdpBridgeManager, + stateManager, + taskMemory: {}, + } as unknown as ComputerUseServerRuntime + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'click', input: { x: 10, y: 20, captureAfter: false } }, 'desktop_click') + + const summaryText = result.content.find(item => item.type === 'text')?.text ?? '' + expect(summaryText).toContain('browser_dom') + expect(runtime.stateManager.getState().browserSurfaceAvailability).toMatchObject({ + preferredSurface: 'browser_dom', + selectedToolName: 'browser_dom_read_page', + }) + expect(cdpBridgeManager.probeAvailability).toHaveBeenCalledTimes(1) + + const structured = result.structuredContent as Record + expect(structured.transparency.advisories).toContainEqual(expect.objectContaining({ + kind: 'use_browser_surface', + reason: expect.stringContaining('extension DOM stack is preferred'), + })) + }) +}) diff --git a/services/computer-use-mcp/src/server/action-executor.ts b/services/computer-use-mcp/src/server/action-executor.ts new file mode 100644 index 000000000..d7e367c94 --- /dev/null +++ b/services/computer-use-mcp/src/server/action-executor.ts @@ -0,0 +1,577 @@ +import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js' + +import type { + ActionInvocation, + ComputerUseConfig, + DesktopExecutor, + PolicyDecision, + ScreenshotArtifact, + TerminalCommandResult, + TerminalState, +} from '../types' +import type { ComputerUseServerRuntime } from './runtime' + +import { normalizeConfiguredAppAction } from '../app-aliases' +import { evaluateActionPolicy } from '../policy' +import { getRuntimePreflight } from '../preflight' +import { buildCoordinateSpaceInfo } from '../runtime-probes' +import { evaluateStrategy, summarizeAdvisories } from '../strategy' +import { buildPointerTrace } from '../trace' +import { + explainActionIntent, + explainActionOutcome, + explainApprovalReason, +} from '../transparency' +import { + maskClipboardPreview, + readClipboardText, + writeClipboardText, +} from '../utils/clipboard' +import { + maskEnvValuePreview, + readEnvValue, +} from '../utils/env-file' +import { describeExecutionTarget } from './formatters' +import { refreshRuntimeRunState } from './refresh-run-state' +import { + buildApprovalResponse, + buildDeniedResponse, + buildExecutionErrorResponse, + buildSuccessResponse, +} from './responses' + +export interface ExecuteActionOptions { + skipApprovalQueue?: boolean +} + +export type ExecuteAction = (action: ActionInvocation, toolName: string, options?: ExecuteActionOptions) => Promise + +function isMutatingAction(action: ActionInvocation) { + return !['screenshot', 'observe_windows', 'wait', 'terminal_reset', 'clipboard_read_text', 'secret_read_env_value'].includes(action.kind) +} + +async function captureOptionalScreenshot(params: { + action: ActionInvocation + executor: DesktopExecutor + config: ComputerUseConfig +}) { + let captureAfter = params.config.defaultCaptureAfter + + switch (params.action.kind) { + case 'click': + case 'type_text': + case 'press_keys': + case 'scroll': + case 'wait': + captureAfter = params.action.input.captureAfter ?? params.config.defaultCaptureAfter + break + case 'screenshot': + captureAfter = true + break + default: + captureAfter = false + break + } + + if (!captureAfter) + return undefined + + return await params.executor.takeScreenshot({ + label: `${params.action.kind}-after`, + }) +} + +function buildDeniedDecision(params: { + decision: PolicyDecision + issues: string[] +}): PolicyDecision { + return { + ...params.decision, + allowed: false, + reasons: [...params.decision.reasons, ...params.issues], + reason: params.decision.reason || params.issues[0], + } +} + +function toScreenshotContent(screenshot: ScreenshotArtifact) { + return { + path: screenshot.path, + publicUrl: screenshot.publicUrl, + observationRef: screenshot.observationRef, + width: screenshot.width, + height: screenshot.height, + placeholder: screenshot.placeholder ?? false, + note: screenshot.note, + } +} + +function toTerminalStateContent(state: TerminalState) { + return { + effectiveCwd: state.effectiveCwd, + lastExitCode: state.lastExitCode, + lastCommandSummary: state.lastCommandSummary, + approvalSessionActive: state.approvalSessionActive ?? false, + approvalGrantedScope: state.approvalGrantedScope, + } +} + +export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteAction { + return async (action, toolName, options = {}) => { + const normalizedAction = normalizeConfiguredAppAction(action, runtime.config.openableApps) + const { executionTarget, context, displayInfo } = await refreshRuntimeRunState(runtime) + + const budget = runtime.session.getBudgetState() + const preflight = getRuntimePreflight({ + config: runtime.config, + lastScreenshot: runtime.session.getLastScreenshot(), + displayInfo, + executionTarget, + }) + const decision = evaluateActionPolicy({ + action: normalizedAction, + config: runtime.config, + context, + operationsExecuted: budget.operationsExecuted, + operationUnitsConsumed: budget.operationUnitsConsumed, + }) + runtime.stateManager.updatePolicyDecision(decision) + + // Evaluate strategy advisories. + const advisories = evaluateStrategy({ + proposedAction: normalizedAction, + state: runtime.stateManager.getState(), + freshContext: context, + }) + const advisorySummary = summarizeAdvisories(advisories) + + // Build transparency: explain what we're about to do and why. + const intent = explainActionIntent(normalizedAction, runtime.stateManager.getState()) + + await runtime.session.record({ + event: 'requested', + toolName, + action: normalizedAction, + context, + policy: decision, + result: { + executionTarget, + displayInfo, + coordinateSpace: preflight.coordinateSpace, + }, + }) + + if (preflight.blockingIssues.length > 0) { + const deniedDecision = buildDeniedDecision({ + decision, + issues: preflight.blockingIssues, + }) + + await runtime.session.record({ + event: 'denied', + toolName, + action: normalizedAction, + context, + policy: deniedDecision, + result: { + executionTarget, + coordinateSpace: preflight.coordinateSpace, + launchContext: preflight.launchContext, + }, + }) + + return buildDeniedResponse(deniedDecision, context, executionTarget) + } + + if (isMutatingAction(normalizedAction) && preflight.mutationReadinessIssues.length > 0) { + const deniedDecision = buildDeniedDecision({ + decision, + issues: preflight.mutationReadinessIssues, + }) + + await runtime.session.record({ + event: 'denied', + toolName, + action: normalizedAction, + context, + policy: deniedDecision, + result: { + executionTarget, + coordinateSpace: preflight.coordinateSpace, + launchContext: preflight.launchContext, + }, + }) + + return buildDeniedResponse(deniedDecision, context, executionTarget) + } + + if (!decision.allowed) { + await runtime.session.record({ + event: 'denied', + toolName, + action: normalizedAction, + context, + policy: decision, + result: { + executionTarget, + }, + }) + + return buildDeniedResponse(decision, context, executionTarget) + } + + if (decision.requiresApproval && !options.skipApprovalQueue) { + const pending = runtime.session.createPendingAction({ + toolName, + action: normalizedAction, + context, + policy: decision, + }) + runtime.stateManager.setPendingApprovalCount(runtime.session.listPendingActions().length) + + await runtime.session.record({ + event: 'approval_required', + toolName, + action: normalizedAction, + context, + policy: decision, + result: { + executionTarget, + pendingActionId: pending.id, + }, + }) + + // Transparency: explain why approval is needed. + const approvalExplanation = explainApprovalReason(normalizedAction, decision, context) + return buildApprovalResponse(pending, decision, context, { + intent, + approvalReason: approvalExplanation, + advisorySummary, + }) + } + + try { + let backendResult: Record = {} + let clipboardStructuredContent: Record | undefined + let secretStructuredContent: Record | undefined + + switch (normalizedAction.kind) { + case 'screenshot': { + const screenshot = await runtime.executor.takeScreenshot(normalizedAction.input) + runtime.session.setLastScreenshot(screenshot) + runtime.stateManager.updateLastScreenshot({ + path: screenshot.path, + width: screenshot.width, + height: screenshot.height, + capturedAt: screenshot.capturedAt, + placeholder: screenshot.placeholder ?? false, + note: screenshot.note, + executionTargetMode: screenshot.executionTargetMode, + sourceHostName: screenshot.sourceHostName, + sourceDisplayId: screenshot.sourceDisplayId, + sourceSessionTag: screenshot.sourceSessionTag, + }) + runtime.session.consumeOperation(decision.estimatedOperationUnits) + + await runtime.session.record({ + event: 'executed', + toolName, + action: normalizedAction, + context, + policy: decision, + result: { + executionTarget, + screenshotPath: screenshot.path, + width: screenshot.width, + height: screenshot.height, + placeholder: screenshot.placeholder ?? false, + }, + }) + + return buildSuccessResponse({ + summary: `Screenshot captured (${screenshot.width || '?'}x${screenshot.height || '?'}) on ${describeExecutionTarget(executionTarget)}.`, + screenshot, + structuredContent: { + status: 'executed', + action: normalizedAction.kind, + context, + policy: decision, + launchContext: preflight.launchContext, + executionTarget, + displayInfo, + coordinateSpace: buildCoordinateSpaceInfo({ + config: runtime.config, + lastScreenshot: runtime.session.getLastScreenshot(), + displayInfo, + }), + screenshot: toScreenshotContent(screenshot), + }, + }) + } + case 'observe_windows': { + const observation = await runtime.executor.observeWindows(normalizedAction.input) + runtime.stateManager.updateWindowObservation(observation) + backendResult = { observation } + break + } + case 'open_app': { + const result = await runtime.executor.openApp(normalizedAction.input) + backendResult = { + ...result, + app: normalizedAction.input.app, + } + break + } + case 'focus_app': { + const result = await runtime.executor.focusApp(normalizedAction.input) + backendResult = { + ...result, + app: normalizedAction.input.app, + } + break + } + case 'click': { + const pointerTrace = buildPointerTrace({ + from: runtime.session.getPointerPosition(), + to: { x: normalizedAction.input.x, y: normalizedAction.input.y }, + bounds: runtime.config.allowedBounds, + }) + const result = await runtime.executor.click({ + ...normalizedAction.input, + pointerTrace, + }) + runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y }) + backendResult = { + ...result, + pointerTrace, + } + break + } + case 'type_text': { + if (typeof normalizedAction.input.x === 'number' && typeof normalizedAction.input.y === 'number') { + const pointerTrace = buildPointerTrace({ + from: runtime.session.getPointerPosition(), + to: { x: normalizedAction.input.x, y: normalizedAction.input.y }, + bounds: runtime.config.allowedBounds, + }) + // NOTICE: The preparatory click must succeed before we type. + // If focus fails the text would go to the wrong element. + try { + await runtime.executor.click({ + x: normalizedAction.input.x, + y: normalizedAction.input.y, + button: 'left', + clickCount: 1, + pointerTrace, + }) + runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y }) + backendResult.focusPointerTrace = pointerTrace + } + catch (clickError) { + const msg = clickError instanceof Error ? clickError.message : String(clickError) + throw new Error(`Preparatory click at (${normalizedAction.input.x}, ${normalizedAction.input.y}) failed before typing: ${msg}`) + } + } + const result = await runtime.executor.typeText(normalizedAction.input) + backendResult = { + ...backendResult, + ...result, + } + break + } + case 'press_keys': { + const result = await runtime.executor.pressKeys(normalizedAction.input) + backendResult = { ...result } + break + } + case 'scroll': { + const result = await runtime.executor.scroll(normalizedAction.input) + if (typeof normalizedAction.input.x === 'number' && typeof normalizedAction.input.y === 'number') { + runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y }) + } + backendResult = { ...result } + break + } + case 'wait': { + const result = await runtime.executor.wait(normalizedAction.input) + backendResult = { ...result } + break + } + case 'terminal_exec': { + const result = await runtime.terminalRunner.execute(normalizedAction.input) + runtime.session.setTerminalState(runtime.terminalRunner.getState()) + runtime.stateManager.updateTerminalResult(result) + backendResult = { + ...result, + terminalState: toTerminalStateContent(runtime.session.getTerminalState()), + } + break + } + case 'terminal_reset': { + const state = runtime.terminalRunner.resetState(normalizedAction.input.reason) + runtime.session.setTerminalState(state) + backendResult = { + terminalState: toTerminalStateContent(state), + } + break + } + case 'secret_read_env_value': { + const result = await readEnvValue(normalizedAction.input) + backendResult = { + filePath: result.filePath, + key: result.key, + valueLength: result.value.length, + preview: maskEnvValuePreview(result.value), + } + secretStructuredContent = { + filePath: result.filePath, + key: result.key, + value: result.value, + valueLength: result.value.length, + } + break + } + case 'clipboard_read_text': { + const result = await readClipboardText(runtime.config, normalizedAction.input) + backendResult = { + textLength: result.originalLength, + returnedLength: result.returnedLength, + trimmed: result.trimmed, + truncated: result.truncated, + preview: maskClipboardPreview(result.text), + } + clipboardStructuredContent = { + text: result.text, + textLength: result.originalLength, + returnedLength: result.returnedLength, + trimmed: result.trimmed, + truncated: result.truncated, + } + break + } + case 'clipboard_write_text': { + const result = await writeClipboardText(runtime.config, normalizedAction.input.text) + backendResult = { + textLength: result.textLength, + preview: maskClipboardPreview(normalizedAction.input.text), + } + clipboardStructuredContent = { + textLength: result.textLength, + } + break + } + } + + runtime.session.consumeOperation(decision.estimatedOperationUnits) + const screenshot = await captureOptionalScreenshot({ + action: normalizedAction, + executor: runtime.executor, + config: runtime.config, + }) + if (screenshot) { + runtime.session.setLastScreenshot(screenshot) + runtime.stateManager.updateLastScreenshot({ + path: screenshot.path, + width: screenshot.width, + height: screenshot.height, + capturedAt: screenshot.capturedAt, + placeholder: screenshot.placeholder ?? false, + note: screenshot.note, + executionTargetMode: screenshot.executionTargetMode, + sourceHostName: screenshot.sourceHostName, + sourceDisplayId: screenshot.sourceDisplayId, + sourceSessionTag: screenshot.sourceSessionTag, + }) + } + + // Transparency: explain what just happened. + const outcome = explainActionOutcome({ + action: normalizedAction, + succeeded: true, + terminalResult: normalizedAction.kind === 'terminal_exec' ? (backendResult as unknown as TerminalCommandResult) : undefined, + context, + }) + + await runtime.session.record({ + event: 'executed', + toolName, + action: normalizedAction, + context, + policy: decision, + result: { + ...backendResult, + executionTarget, + screenshotPath: screenshot?.path, + displayInfo, + }, + }) + + return buildSuccessResponse({ + summary: `${intent} ${outcome}${advisorySummary ? ` Strategy: ${advisorySummary}` : ''}`, + screenshot, + structuredContent: { + status: 'executed', + action: normalizedAction.kind, + context, + policy: decision, + launchContext: preflight.launchContext, + executionTarget, + displayInfo, + coordinateSpace: buildCoordinateSpaceInfo({ + config: runtime.config, + lastScreenshot: runtime.session.getLastScreenshot(), + displayInfo, + }), + backendResult, + secret: secretStructuredContent, + clipboard: clipboardStructuredContent, + terminalState: normalizedAction.kind.startsWith('terminal_') ? toTerminalStateContent(runtime.session.getTerminalState()) : undefined, + screenshot: screenshot + ? toScreenshotContent(screenshot) + : undefined, + // Transparency fields. + transparency: { + intent, + outcome, + advisories: advisories.map(a => ({ kind: a.kind, reason: a.reason })), + }, + }, + }) + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + + // Update run state with failure info. + if (runtime.stateManager.hasActiveTask()) { + runtime.stateManager.completeCurrentStep('failure', errorMessage) + } + + // Transparency: explain what failed. + const failureExplanation = explainActionOutcome({ + action: normalizedAction, + succeeded: false, + errorMessage, + context, + }) + + await runtime.session.record({ + event: 'failed', + toolName, + action: normalizedAction, + context, + policy: decision, + result: { + executionTarget, + error: errorMessage, + }, + }) + + return buildExecutionErrorResponse({ + errorMessage: `${failureExplanation}${advisorySummary ? ` Strategy: ${advisorySummary}` : ''}`, + action: normalizedAction, + context, + executionTarget, + policy: decision, + }) + } + } +} diff --git a/services/computer-use-mcp/src/server/browser-agent.ts b/services/computer-use-mcp/src/server/browser-agent.ts new file mode 100644 index 000000000..789ed5e73 --- /dev/null +++ b/services/computer-use-mcp/src/server/browser-agent.ts @@ -0,0 +1,189 @@ +import { spawn } from 'node:child_process' +import { existsSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { env } from 'node:process' +import { fileURLToPath } from 'node:url' + +export interface BrowserAgentTaskInput { + instruction: string + agent?: 'google' | 'kimi' + cdpUrl?: string + maxTurns?: number + timeoutMs?: number +} + +export interface BrowserAgentLaunchContext { + cdpUrl: string + cliCwd: string + cliModule: string + pythonCommand: string + rootExists: boolean +} + +export interface BrowserAgentTaskResult { + success: boolean + instruction: string + agent: 'google' | 'kimi' + cdpUrl: string + cliCwd: string + cliModule: string + pythonCommand: string + exitCode: number | null + timedOut: boolean + stderrLines: string[] + payload?: Record +} + +const computerUseRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../bin/computer_use') +const cliModule = 'google_computer_use.cli' + +function trimNonEmptyLines(text: string) { + return text + .split(/\r?\n/u) + .map(line => line.trim()) + .filter(Boolean) +} + +function resolvePythonCommand(root: string) { + const configured = env.COMPUTER_USE_PYTHON?.trim() + if (configured) { + return configured + } + + const venvPython = resolve(root, '.venv', 'bin', 'python') + if (existsSync(venvPython)) { + return venvPython + } + + return 'python3' +} + +export function getBrowserAgentLaunchContext(overrides?: { + cdpUrl?: string +}): BrowserAgentLaunchContext { + const cliCwd = env.COMPUTER_USE_BROWSER_AGENT_ROOT?.trim() + ? resolve(env.COMPUTER_USE_BROWSER_AGENT_ROOT) + : computerUseRoot + + return { + cdpUrl: overrides?.cdpUrl?.trim() || env.CDP_URL?.trim() || 'http://localhost:9222', + cliCwd, + cliModule, + pythonCommand: resolvePythonCommand(cliCwd), + rootExists: existsSync(cliCwd), + } +} + +export async function runBrowserAgentTask(input: BrowserAgentTaskInput): Promise { + const instruction = input.instruction.trim() + if (!instruction) { + throw new Error('browser agent requires a non-empty instruction') + } + + const agent = input.agent || 'google' + const maxTurns = Math.max(1, Math.min(input.maxTurns || 30, 80)) + const timeoutMs = Math.max(1_000, Math.min(input.timeoutMs || 180_000, 900_000)) + const launchContext = getBrowserAgentLaunchContext({ cdpUrl: input.cdpUrl }) + + if (!launchContext.rootExists) { + throw new Error(`browser agent root does not exist: ${launchContext.cliCwd}`) + } + + return await new Promise((resolvePromise, rejectPromise) => { + const child = spawn(launchContext.pythonCommand, ['-m', launchContext.cliModule], { + cwd: launchContext.cliCwd, + env: { + ...env, + CDP_URL: launchContext.cdpUrl, + }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + + let stdout = '' + let stderr = '' + let settled = false + let timedOut = false + + const timeoutHandle = setTimeout(() => { + timedOut = true + child.kill('SIGTERM') + setTimeout(() => { + if (child.exitCode == null) { + child.kill('SIGKILL') + } + }, 1_500) + }, timeoutMs) + + const finish = (callback: () => void) => { + if (settled) { + return + } + + settled = true + clearTimeout(timeoutHandle) + callback() + } + + child.stdout.on('data', (chunk) => { + stdout += chunk.toString('utf-8') + }) + + child.stderr.on('data', (chunk) => { + stderr += chunk.toString('utf-8') + }) + + child.once('error', (error) => { + finish(() => { + rejectPromise(error) + }) + }) + + child.once('close', (code) => { + finish(() => { + const stdoutLines = trimNonEmptyLines(stdout) + const stderrLines = trimNonEmptyLines(stderr) + const rawPayload = stdoutLines.at(-1) + + let payload: Record | undefined + if (rawPayload) { + try { + const parsed = JSON.parse(rawPayload) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + payload = parsed as Record + } + } + catch { + // Ignore parse failure here; the caller still gets stderr and exit code. + } + } + + if (!payload && code !== 0) { + rejectPromise(new Error(stderrLines.at(-1) || `browser agent exited with code ${String(code)}`)) + return + } + + resolvePromise({ + success: Boolean(payload?.success), + instruction, + agent, + cdpUrl: launchContext.cdpUrl, + cliCwd: launchContext.cliCwd, + cliModule: launchContext.cliModule, + pythonCommand: launchContext.pythonCommand, + exitCode: code, + timedOut, + stderrLines, + payload, + }) + }) + }) + + child.stdin.write(`${JSON.stringify({ + instruction, + agent, + cdp_url: launchContext.cdpUrl, + max_turns: maxTurns, + })}\n`) + child.stdin.end() + }) +} diff --git a/services/computer-use-mcp/src/server/browser-surface.ts b/services/computer-use-mcp/src/server/browser-surface.ts new file mode 100644 index 000000000..4a63dfdaa --- /dev/null +++ b/services/computer-use-mcp/src/server/browser-surface.ts @@ -0,0 +1,76 @@ +import type { + BrowserDomBridgeStatus, + BrowserSurfaceAvailability, + ExecutionTarget, +} from '../types' +import type { CdpAvailabilityStatus } from './cdp-manager' + +export function buildBrowserSurfaceAvailability(params: { + executionTarget?: ExecutionTarget + extension: BrowserDomBridgeStatus + cdp: CdpAvailabilityStatus +}): BrowserSurfaceAvailability { + const { executionTarget, extension, cdp } = params + const executionMode = executionTarget?.mode ?? 'dry-run' + const suitable = executionMode !== 'remote' + + const availableSurfaces: BrowserSurfaceAvailability['availableSurfaces'] = [] + if (suitable && extension.enabled && extension.connected) { + availableSurfaces.push('browser_dom') + } + if (suitable && (cdp.connected || cdp.connectable)) { + availableSurfaces.push('browser_cdp') + } + + let preferredSurface: BrowserSurfaceAvailability['preferredSurface'] + let selectedToolName: BrowserSurfaceAvailability['selectedToolName'] + let reason: string + + if (!suitable) { + reason = 'Browser DOM/CDP surfaces are not suitable when the execution target is remote desktop.' + } + else if (extension.enabled && extension.connected) { + preferredSurface = 'browser_dom' + selectedToolName = 'browser_dom_read_page' + reason = 'Browser extension bridge is already connected, so the extension DOM stack is preferred.' + } + else if (cdp.connected) { + preferredSurface = 'browser_cdp' + selectedToolName = 'browser_cdp_collect_elements' + reason = 'CDP is already connected and ready to inspect the current browser page.' + } + else if (cdp.connectable) { + preferredSurface = 'browser_cdp' + selectedToolName = 'browser_cdp_collect_elements' + reason = 'Browser extension is unavailable, but the CDP endpoint is reachable and can be used as fallback.' + } + else if (extension.lastError) { + reason = `Browser extension bridge is unavailable: ${extension.lastError}` + } + else if (cdp.lastError) { + reason = `CDP endpoint is unavailable: ${cdp.lastError}` + } + else { + reason = 'No browser surface bridge is currently available.' + } + + return { + executionMode, + suitable, + availableSurfaces, + preferredSurface, + selectedToolName, + reason, + extension: { + enabled: extension.enabled, + connected: extension.connected, + lastError: extension.lastError, + }, + cdp: { + endpoint: cdp.endpoint, + connected: cdp.connected, + connectable: cdp.connectable, + lastError: cdp.lastError, + }, + } +} diff --git a/services/computer-use-mcp/src/server/cdp-manager.ts b/services/computer-use-mcp/src/server/cdp-manager.ts new file mode 100644 index 000000000..e51c39e01 --- /dev/null +++ b/services/computer-use-mcp/src/server/cdp-manager.ts @@ -0,0 +1,95 @@ +import type { CdpBridgeStatus } from '../browser-dom/cdp-bridge' +import type { ComputerUseConfig } from '../types' + +import { CdpBridge } from '../browser-dom/cdp-bridge' + +const DEFAULT_CDP_URL = 'http://localhost:9222' + +export interface CdpAvailabilityStatus { + endpoint: string + connected: boolean + connectable: boolean + lastError?: string +} + +export interface CdpBridgeManager { + ensureBridge: (cdpUrl?: string) => Promise + getStatus: () => CdpBridgeStatus + probeAvailability: (cdpUrl?: string) => Promise + close: () => Promise +} + +export function createCdpBridgeManager(config: ComputerUseConfig): CdpBridgeManager { + let cdpBridge: CdpBridge | undefined + let lastRequestedUrl = DEFAULT_CDP_URL + + return { + async ensureBridge(cdpUrl?: string) { + const url = cdpUrl || DEFAULT_CDP_URL + lastRequestedUrl = url + + if (cdpBridge && cdpBridge.getStatus().connected && cdpBridge.getStatus().cdpUrl === url) { + return cdpBridge + } + + if (cdpBridge) { + await cdpBridge.close() + } + + cdpBridge = new CdpBridge({ + cdpUrl: url, + requestTimeoutMs: config.browserDomBridge.requestTimeoutMs, + }) + + await cdpBridge.connect() + return cdpBridge + }, + + getStatus() { + return cdpBridge?.getStatus() ?? { + cdpUrl: lastRequestedUrl, + connected: false, + } + }, + + async probeAvailability(cdpUrl?: string) { + const endpoint = cdpUrl || lastRequestedUrl || DEFAULT_CDP_URL + const connected = Boolean(cdpBridge?.getStatus().connected && cdpBridge.getStatus().cdpUrl === endpoint) + + try { + const response = await fetch(`${endpoint}/json/list`) + + if (!response.ok) { + throw new Error(`HTTP ${response.status} ${response.statusText}`) + } + + const targets = await response.json() as Array<{ type?: string, webSocketDebuggerUrl?: string }> + const connectable = targets.some(target => target.type === 'page' && typeof target.webSocketDebuggerUrl === 'string' && target.webSocketDebuggerUrl.length > 0) + + return { + endpoint, + connected, + connectable, + lastError: connectable ? undefined : 'No page target with WebSocket debugger URL was found.', + } + } + catch (error) { + return { + endpoint, + connected, + connectable: false, + lastError: error instanceof Error ? error.message : String(error), + } + } + }, + + async close() { + if (!cdpBridge) { + return + } + + await cdpBridge.close() + cdpBridge = undefined + }, + } +} diff --git a/services/computer-use-mcp/src/server/content.ts b/services/computer-use-mcp/src/server/content.ts new file mode 100644 index 000000000..94fd7c379 --- /dev/null +++ b/services/computer-use-mcp/src/server/content.ts @@ -0,0 +1,18 @@ +import type { ImageContent, TextContent } from '@modelcontextprotocol/sdk/types.js' + +import type { ScreenshotArtifact } from '../types' + +export function textContent(text: string): TextContent { + return { + type: 'text', + text, + } +} + +export function imageContent(screenshot: ScreenshotArtifact): ImageContent { + return { + type: 'image', + data: screenshot.dataBase64, + mimeType: screenshot.mimeType, + } +} diff --git a/services/computer-use-mcp/src/server/formatters.ts b/services/computer-use-mcp/src/server/formatters.ts new file mode 100644 index 000000000..e0389ba13 --- /dev/null +++ b/services/computer-use-mcp/src/server/formatters.ts @@ -0,0 +1,30 @@ +import type { CoordinateSpaceInfo, ExecutionTarget, PolicyDecision } from '../types' + +export function describeForegroundContext(record: { appName?: string, windowTitle?: string, available: boolean }) { + if (!record.available) + return 'foreground context unavailable' + + return `${record.appName || 'unknown app'}${record.windowTitle ? ` / ${record.windowTitle}` : ''}` +} + +export function describePolicy(decision: PolicyDecision) { + const state = decision.allowed ? 'allowed' : 'denied' + return `${state}, risk=${decision.riskLevel}, units=${decision.estimatedOperationUnits}${decision.requiresApproval ? ', approval required' : ''}` +} + +export function describeExecutionTarget(target: ExecutionTarget) { + if (target.mode === 'dry-run') + return `local dry-run on ${target.hostName}` + if (target.mode === 'local-windowed') + return `local macOS windowed execution on ${target.hostName}` + + return `${target.hostName}${target.displayId ? ` ${target.displayId}` : ''}${target.sessionTag ? ` (${target.sessionTag})` : ''}` +} + +export function summarizeCoordinateSpace(info: CoordinateSpaceInfo) { + if (info.aligned === true) + return 'aligned' + if (info.aligned === false) + return 'mismatch' + return 'unknown' +} diff --git a/services/computer-use-mcp/src/server/refresh-run-state.ts b/services/computer-use-mcp/src/server/refresh-run-state.ts new file mode 100644 index 000000000..687caf282 --- /dev/null +++ b/services/computer-use-mcp/src/server/refresh-run-state.ts @@ -0,0 +1,38 @@ +import type { ComputerUseServerRuntime } from './runtime' + +import { buildBrowserSurfaceAvailability } from './browser-surface' + +export async function refreshRuntimeRunState(runtime: ComputerUseServerRuntime) { + const [executionTarget, context, displayInfo, cdpAvailability] = await Promise.all([ + runtime.executor.getExecutionTarget(), + runtime.executor.getForegroundContext(), + runtime.executor.getDisplayInfo(), + runtime.cdpBridgeManager.probeAvailability(), + ]) + + runtime.stateManager.updateForegroundContext(context) + runtime.stateManager.updateExecutionTarget(executionTarget) + runtime.stateManager.updateDisplayInfo(displayInfo) + runtime.stateManager.updateTerminalState(runtime.terminalRunner.getState()) + runtime.stateManager.setPendingApprovalCount(runtime.session.listPendingActions().length) + + const browserSurfaceAvailability = buildBrowserSurfaceAvailability({ + executionTarget, + extension: runtime.browserDomBridge.getStatus(), + cdp: cdpAvailability, + }) + runtime.stateManager.updateBrowserSurfaceAvailability(browserSurfaceAvailability) + + const lastScreenshot = runtime.session.getLastScreenshot() + if (lastScreenshot) { + runtime.stateManager.updateLastScreenshot(lastScreenshot) + } + + return { + executionTarget, + context, + displayInfo, + cdpAvailability, + browserSurfaceAvailability, + } +} diff --git a/services/computer-use-mcp/src/server/register-accessibility.ts b/services/computer-use-mcp/src/server/register-accessibility.ts new file mode 100644 index 000000000..9d301f259 --- /dev/null +++ b/services/computer-use-mcp/src/server/register-accessibility.ts @@ -0,0 +1,141 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { z } from 'zod' + +import { captureAXTree, formatAXSnapshotAsText } from '../accessibility' +import { textContent } from './content' + +export interface RegisterAccessibilityToolsOptions { + server: McpServer + runtime: ComputerUseServerRuntime +} + +export function registerAccessibilityTools({ server, runtime }: RegisterAccessibilityToolsOptions) { + server.tool( + 'accessibility_snapshot', + { + pid: z.number().int().min(1).optional().describe('Target a specific process by PID; defaults to the frontmost application'), + maxDepth: z.number().int().min(1).max(30).optional().describe('Maximum tree depth to traverse (default: 15)'), + maxNodes: z.number().int().min(1).max(10000).optional().describe('Maximum total nodes to collect (default: 2000)'), + verbose: z.boolean().optional().describe('Include all nodes, even those with empty roles/titles'), + includeBounds: z.boolean().optional().describe('Include screen-coordinate bounding rects in the text output'), + }, + async ({ pid, maxDepth, maxNodes, verbose, includeBounds }) => { + try { + const snapshot = await captureAXTree(runtime.config, { + pid, + maxDepth, + maxNodes, + verbose, + }) + + const text = formatAXSnapshotAsText(snapshot, { + includeBounds: includeBounds ?? false, + includeUids: true, + }) + + return { + content: [ + textContent(text), + ], + structuredContent: { + status: 'ok', + appName: snapshot.appName, + pid: snapshot.pid, + snapshotId: snapshot.snapshotId, + nodeCount: snapshot.uidToNode.size, + truncated: snapshot.truncated, + capturedAt: snapshot.capturedAt, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`Accessibility snapshot failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'accessibility_find_element', + { + role: z.string().optional().describe('AX role to search for, e.g. AXButton, AXTextField'), + title: z.string().optional().describe('Title substring to match (case-insensitive)'), + pid: z.number().int().min(1).optional().describe('Target process PID; defaults to frontmost app'), + maxResults: z.number().int().min(1).max(50).optional().describe('Maximum matches to return (default: 10)'), + }, + async ({ role, title, pid, maxResults }) => { + try { + const snapshot = await captureAXTree(runtime.config, { + pid, + maxDepth: 20, + maxNodes: 5000, + verbose: true, + }) + + const limit = maxResults ?? 10 + const matches: Array<{ + uid: string + role: string + title?: string + value?: string + bounds?: { x: number, y: number, width: number, height: number } + }> = [] + + const titleLower = title?.toLowerCase() + + for (const [uid, node] of snapshot.uidToNode) { + if (matches.length >= limit) + break + + const roleMatch = !role || node.role === role + const titleMatch = !titleLower || (node.title?.toLowerCase().includes(titleLower)) + + if (roleMatch && titleMatch) { + matches.push({ + uid, + role: node.role, + title: node.title, + value: node.value, + bounds: node.bounds, + }) + } + } + + return { + content: [ + textContent(`Found ${matches.length} element(s) matching role=${role ?? 'any'}, title=${title ?? 'any'} in ${snapshot.appName}.`), + ], + structuredContent: { + status: 'ok', + appName: snapshot.appName, + pid: snapshot.pid, + matches, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`Accessibility find failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) +} diff --git a/services/computer-use-mcp/src/server/register-cdp.ts b/services/computer-use-mcp/src/server/register-cdp.ts new file mode 100644 index 000000000..01480357f --- /dev/null +++ b/services/computer-use-mcp/src/server/register-cdp.ts @@ -0,0 +1,269 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { z } from 'zod' + +import { textContent } from './content' + +export interface RegisterCdpToolsOptions { + server: McpServer + runtime: ComputerUseServerRuntime +} + +/** + * Register CDP-based browser tools that connect directly to Chrome + * via the DevTools Protocol, without requiring the browser extension. + */ +export function registerCdpTools({ server, runtime }: RegisterCdpToolsOptions) { + server.tool( + 'browser_cdp_connect', + { + cdpUrl: z.string().optional().describe('Chrome DevTools Protocol endpoint (default: http://localhost:9222)'), + }, + async ({ cdpUrl }) => { + try { + const bridge = await runtime.cdpBridgeManager.ensureBridge(cdpUrl) + const status = bridge.getStatus() + + return { + content: [ + textContent(`CDP connected to ${status.pageTitle} (${status.pageUrl}).`), + ], + structuredContent: { + status: 'ok', + cdp: status, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`CDP connect failed: ${error instanceof Error ? error.message : String(error)}. Ensure Chrome is running with --remote-debugging-port=9222.`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'browser_cdp_status', + {}, + async () => { + const status = runtime.cdpBridgeManager.getStatus() + + return { + content: [ + textContent(`CDP bridge: ${status.connected ? `connected to ${status.pageTitle}` : 'disconnected'}.`), + ], + structuredContent: { + status: 'ok', + cdp: status, + }, + } + }, + ) + + server.tool( + 'browser_cdp_accessibility_snapshot', + { + cdpUrl: z.string().optional().describe('CDP endpoint override'), + }, + async ({ cdpUrl }) => { + try { + const bridge = await runtime.cdpBridgeManager.ensureBridge(cdpUrl) + const snapshot = await bridge.getAccessibilityTree() + const text = bridge.formatAXTreeAsText(snapshot) + + return { + content: [textContent(text)], + structuredContent: { + status: 'ok', + pageUrl: snapshot.pageUrl, + pageTitle: snapshot.pageTitle, + nodeCount: snapshot.nodes.length, + capturedAt: snapshot.capturedAt, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`CDP accessibility snapshot failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'browser_cdp_evaluate', + { + expression: z.string().min(1).describe('JavaScript expression to evaluate in the page context'), + cdpUrl: z.string().optional().describe('CDP endpoint override'), + }, + async ({ expression, cdpUrl }) => { + try { + const bridge = await runtime.cdpBridgeManager.ensureBridge(cdpUrl) + const result = await bridge.evaluate(expression) + + return { + content: [textContent(typeof result === 'string' ? result : JSON.stringify(result, null, 2))], + structuredContent: { + status: 'ok', + result, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`CDP evaluate failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'browser_cdp_collect_elements', + { + maxElements: z.number().int().min(1).max(500).optional().describe('Maximum interactive elements to collect (default: 200)'), + cdpUrl: z.string().optional().describe('CDP endpoint override'), + }, + async ({ maxElements, cdpUrl }) => { + try { + const bridge = await runtime.cdpBridgeManager.ensureBridge(cdpUrl) + const elements = await bridge.collectInteractiveElements(maxElements) + + return { + content: [ + textContent(`Collected ${elements.length} interactive element(s) from ${bridge.getStatus().pageTitle}.`), + ], + structuredContent: { + status: 'ok', + elementCount: elements.length, + elements, + page: { + url: bridge.getStatus().pageUrl, + title: bridge.getStatus().pageTitle, + }, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`CDP collect elements failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'browser_cdp_screenshot', + { + format: z.enum(['png', 'jpeg']).optional().describe('Image format (default: png)'), + quality: z.number().int().min(0).max(100).optional().describe('JPEG quality (only for jpeg format)'), + cdpUrl: z.string().optional().describe('CDP endpoint override'), + }, + async ({ format, quality, cdpUrl }) => { + try { + const bridge = await runtime.cdpBridgeManager.ensureBridge(cdpUrl) + const base64 = await bridge.screenshot({ format, quality }) + + return { + content: [ + { + type: 'image' as const, + data: base64, + mimeType: format === 'jpeg' ? 'image/jpeg' : 'image/png', + }, + ], + structuredContent: { + status: 'ok', + format: format ?? 'png', + page: { + url: bridge.getStatus().pageUrl, + title: bridge.getStatus().pageTitle, + }, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`CDP screenshot failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'browser_cdp_navigate', + { + url: z.string().min(1).describe('URL to navigate to'), + cdpUrl: z.string().optional().describe('CDP endpoint override'), + }, + async ({ url, cdpUrl }) => { + try { + const bridge = await runtime.cdpBridgeManager.ensureBridge(cdpUrl) + await bridge.navigate(url) + + return { + content: [textContent(`Navigated to ${url}.`)], + structuredContent: { + status: 'ok', + url, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`CDP navigate failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + // Return a cleanup function for server shutdown + return { + async close() { + await runtime.cdpBridgeManager.close() + }, + } +} diff --git a/services/computer-use-mcp/src/server/register-display.ts b/services/computer-use-mcp/src/server/register-display.ts new file mode 100644 index 000000000..367abc26d --- /dev/null +++ b/services/computer-use-mcp/src/server/register-display.ts @@ -0,0 +1,122 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { z } from 'zod' + +import { enumerateDisplays, findDisplayForPoint, formatDisplaySummary } from '../display' +import { textContent } from './content' + +export interface RegisterDisplayToolsOptions { + server: McpServer + runtime: ComputerUseServerRuntime +} + +export function registerDisplayTools({ server, runtime }: RegisterDisplayToolsOptions) { + server.tool( + 'display_enumerate', + {}, + async () => { + try { + const snapshot = await enumerateDisplays(runtime.config) + const summary = formatDisplaySummary(snapshot) + + return { + content: [ + textContent(summary), + ], + structuredContent: { + status: 'ok', + displayCount: snapshot.displays.length, + displays: snapshot.displays.map(d => ({ + displayId: d.displayId, + isMain: d.isMain, + isBuiltIn: d.isBuiltIn, + bounds: d.bounds, + visibleBounds: d.visibleBounds, + scaleFactor: d.scaleFactor, + pixelWidth: d.pixelWidth, + pixelHeight: d.pixelHeight, + })), + combinedBounds: snapshot.combinedBounds, + capturedAt: snapshot.capturedAt, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`Display enumeration failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'display_identify_point', + { + x: z.number().describe('Logical X coordinate in global screen space'), + y: z.number().describe('Logical Y coordinate in global screen space'), + }, + async ({ x, y }) => { + try { + const snapshot = await enumerateDisplays(runtime.config) + const display = findDisplayForPoint(snapshot, x, y) + + if (!display) { + return { + content: [ + textContent(`Point (${x}, ${y}) is outside all connected displays.`), + ], + structuredContent: { + status: 'outside', + point: { x, y }, + displays: snapshot.displays.map(d => ({ + displayId: d.displayId, + bounds: d.bounds, + })), + }, + } + } + + return { + content: [ + textContent(`Point (${x}, ${y}) is on display #${display.displayId}${display.isMain ? ' (main)' : ''} — ${display.bounds.width}x${display.bounds.height}.`), + ], + structuredContent: { + status: 'ok', + point: { x, y }, + display: { + displayId: display.displayId, + isMain: display.isMain, + bounds: display.bounds, + scaleFactor: display.scaleFactor, + }, + localCoord: { + x: x - display.bounds.x, + y: y - display.bounds.y, + }, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`Display identify failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) +} diff --git a/services/computer-use-mcp/src/server/register-pty-terminal-lane.test.ts b/services/computer-use-mcp/src/server/register-pty-terminal-lane.test.ts new file mode 100644 index 000000000..e95cca4ae --- /dev/null +++ b/services/computer-use-mcp/src/server/register-pty-terminal-lane.test.ts @@ -0,0 +1,497 @@ +/** + * Terminal Lane v1 — PTY tool registration tests. + * + * Tests the terminal lane behavior of register-pty tools: + * - Open Grant lifecycle: create issues grant, destroy revokes + * - Audit logging: every operation logged, send_input only byte count + preview + * - stepId binding: pty_create binds to stepId + * - pty_send_input primary name + pty_write compat alias + */ + +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { RunStateManager } from '../state' +import { + createPtySession, + destroyPtySession, + isPtyAvailable, + listPtySessions, + readPtyScreen, + resizePty, + writeToPty, +} from '../terminal/pty-runner' +import { createTestConfig } from '../test-fixtures' +import { createAcquirePtyCallback, executeApprovedPtyCreate, registerPtyTools } from './register-pty' + +vi.mock('../terminal/pty-runner', () => ({ + createPtySession: vi.fn(), + destroyAllPtySessions: vi.fn(), + destroyPtySession: vi.fn(), + getPtyAvailabilityInfo: vi.fn().mockResolvedValue({ available: true }), + isPtyAvailable: vi.fn(), + listPtySessions: vi.fn(), + readPtyScreen: vi.fn(), + resizePty: vi.fn(), + writeToPty: vi.fn(), +})) + +type ToolHandler = (args: Record) => Promise + +function createMockServer() { + const handlers = new Map() + + return { + server: { + tool(name: string, _schema: unknown, handler: ToolHandler) { + handlers.set(name, handler) + }, + } as unknown as McpServer, + async invoke(name: string, args: Record = {}) { + const handler = handlers.get(name) + if (!handler) { + throw new Error(`Missing registered tool: ${name}`) + } + + return await handler(args) + }, + hasHandler(name: string) { + return handlers.has(name) + }, + } +} + +describe('register-pty: terminal lane', () => { + let runtime: ComputerUseServerRuntime + let pendingActions: Array> + + beforeEach(() => { + pendingActions = [] + runtime = { + config: createTestConfig({ approvalMode: 'never' }), + stateManager: new RunStateManager(), + session: { + createPendingAction: vi.fn((record: Record) => { + const pending = { ...record, id: `pending_${pendingActions.length + 1}`, createdAt: new Date().toISOString() } + pendingActions.push(pending) + return pending + }), + listPendingActions: vi.fn(() => pendingActions), + record: vi.fn().mockResolvedValue(undefined), + }, + } as unknown as ComputerUseServerRuntime + vi.clearAllMocks() + }) + + // ----------------------------------------------------------------------- + // Open Grant lifecycle + // ----------------------------------------------------------------------- + + describe('open grant lifecycle', () => { + it('pty_create returns approval_required when approvals are enabled', async () => { + runtime.config = createTestConfig({ approvalMode: 'actions' }) + vi.mocked(isPtyAvailable).mockResolvedValue(true) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_create', { + rows: 24, + cols: 80, + cwd: '/tmp', + approvalSessionId: 'approval_1', + }) + const structured = result.structuredContent as Record + + expect(structured.status).toBe('approval_required') + expect((runtime.session.createPendingAction as any)).toHaveBeenCalledTimes(1) + expect(runtime.stateManager.getActivePtyGrants()).toHaveLength(0) + }) + + it('approved PTY create issues an Open Grant', async () => { + runtime.config = createTestConfig({ approvalMode: 'actions' }) + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 1000, + }) + const result = await executeApprovedPtyCreate(runtime, { + rows: 24, + cols: 80, + approvalSessionId: 'approval_1', + }) + + expect((result.structuredContent as Record).approvalSessionId).toBe('approval_1') + expect(runtime.stateManager.getActivePtyGrants()).toHaveLength(1) + }) + + it('workflow PTY self-acquire queues approval with a grant session id', async () => { + runtime.config = createTestConfig({ approvalMode: 'actions' }) + vi.mocked(isPtyAvailable).mockResolvedValue(true) + const acquirePty = createAcquirePtyCallback(runtime) + + const result = await acquirePty({ + taskId: 'task_terminal_lane', + stepId: 'step_terminal_lane', + cwd: '/tmp/project', + rows: 24, + cols: 80, + autoApprove: false, + }) + + expect(result).toMatchObject({ + acquired: false, + approvalPending: true, + }) + expect(pendingActions).toHaveLength(1) + expect(pendingActions[0]).toMatchObject({ + toolName: 'pty_create', + action: { + kind: 'pty_create', + input: expect.objectContaining({ + cwd: '/tmp/project', + stepId: 'step_terminal_lane', + approvalSessionId: expect.any(String), + }), + }, + }) + }) + + it('pty_destroy revokes the Open Grant', async () => { + runtime.config = createTestConfig({ approvalMode: 'actions' }) + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 1000, + }) + vi.mocked(destroyPtySession).mockReturnValue(true) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await executeApprovedPtyCreate(runtime, { rows: 24, cols: 80, approvalSessionId: 'approval_1' }) + expect(runtime.stateManager.getActivePtyGrants()).toHaveLength(1) + + await invoke('pty_destroy', { sessionId: 'pty_1', approvalSessionId: 'approval_1' }) + expect(runtime.stateManager.getActivePtyGrants()).toHaveLength(0) + }) + }) + + // ----------------------------------------------------------------------- + // Audit logging + // ----------------------------------------------------------------------- + + describe('audit logging', () => { + it('pty_create writes a create audit entry', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 30, + cols: 120, + screenContent: '', + pid: 2000, + }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await invoke('pty_create', { rows: 30, cols: 120, cwd: '/home/user' }) + + const log = runtime.stateManager.getPtyAuditForSession('pty_1') + expect(log).toHaveLength(1) + expect(log[0].event).toBe('create') + expect(log[0].cwd).toBe('/home/user') + expect(log[0].rows).toBe(30) + expect(log[0].cols).toBe(120) + expect(log[0].pid).toBe(2000) + }) + + it('pty_send_input logs byte count + truncated preview only', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 3000, + }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await invoke('pty_create', { rows: 24, cols: 80 }) + + // Write a long string (> 80 chars) + const longInput = 'a'.repeat(200) + await invoke('pty_send_input', { sessionId: 'pty_1', data: longInput }) + + const inputAudit = runtime.stateManager.getPtyAuditForSession('pty_1') + .filter(e => e.event === 'send_input') + expect(inputAudit).toHaveLength(1) + expect(inputAudit[0].byteCount).toBe(200) + // Preview truncated to 80 chars + ellipsis + expect(inputAudit[0].inputPreview!.length).toBeLessThanOrEqual(81) + }) + + it('pty_read_screen logs line count + alive state', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 3000, + }) + vi.mocked(readPtyScreen).mockReturnValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: 'line1\nline2\nline3', + pid: 3000, + }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await invoke('pty_create', { rows: 24, cols: 80 }) + await invoke('pty_read_screen', { sessionId: 'pty_1' }) + + const readAudit = runtime.stateManager.getPtyAuditForSession('pty_1') + .filter(e => e.event === 'read_screen') + expect(readAudit).toHaveLength(1) + expect(readAudit[0].returnedLineCount).toBe(3) + expect(readAudit[0].alive).toBe(true) + }) + + it('pty_resize logs dimensions', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 3000, + }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await invoke('pty_create', { rows: 24, cols: 80 }) + await invoke('pty_resize', { sessionId: 'pty_1', rows: 40, cols: 160 }) + + const resizeAudit = runtime.stateManager.getPtyAuditForSession('pty_1') + .filter(e => e.event === 'resize') + expect(resizeAudit).toHaveLength(1) + expect(resizeAudit[0].rows).toBe(40) + expect(resizeAudit[0].cols).toBe(160) + }) + + it('pty_destroy logs actor + outcome', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 3000, + }) + vi.mocked(destroyPtySession).mockReturnValue(true) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await invoke('pty_create', { rows: 24, cols: 80 }) + await invoke('pty_destroy', { sessionId: 'pty_1' }) + + const destroyAudit = runtime.stateManager.getPtyAuditLog() + .filter(e => e.event === 'destroy') + expect(destroyAudit).toHaveLength(1) + expect(destroyAudit[0].actor).toBe('tool_call') + expect(destroyAudit[0].outcome).toBe('ok') + }) + }) + + // ----------------------------------------------------------------------- + // stepId binding + // ----------------------------------------------------------------------- + + describe('stepId binding', () => { + it('pty_create binds session to stepId when provided', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 4000, + }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await invoke('pty_create', { stepId: 'step_abc', rows: 24, cols: 80 }) + + const sessions = runtime.stateManager.getPtySessions() + expect(sessions[0].boundStepId).toBe('step_abc') + }) + + it('pty_get_status includes boundStepId in response', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(listPtySessions).mockReturnValue([ + { id: 'pty_1', alive: true, rows: 24, cols: 80, screenContent: '', pid: 5000 }, + ]) + runtime.stateManager.registerPtySession({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + pid: 5000, + }) + runtime.stateManager.bindPtySessionToStepId('pty_1', 'step_xyz') + + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_get_status') + const sessions = (result.structuredContent as Record).sessions + expect(sessions[0].boundStepId).toBe('step_xyz') + }) + }) + + // ----------------------------------------------------------------------- + // pty_send_input + pty_write compat alias + // ----------------------------------------------------------------------- + + describe('pty_send_input / pty_write alias', () => { + it('registers both pty_send_input and pty_write', () => { + const { server, hasHandler } = createMockServer() + registerPtyTools({ server, runtime }) + + expect(hasHandler('pty_send_input')).toBe(true) + expect(hasHandler('pty_write')).toBe(true) + }) + + it('pty_write works identically to pty_send_input', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 6000, + }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + await invoke('pty_create', { rows: 24, cols: 80 }) + const result = await invoke('pty_write', { sessionId: 'pty_1', data: 'ls\r' }) + + expect(writeToPty).toHaveBeenCalledWith('pty_1', { data: 'ls\r' }) + expect((result.structuredContent as Record).status).toBe('ok') + }) + + it('pty_write reports its own operation name in grant errors', async () => { + runtime.config = createTestConfig({ approvalMode: 'actions' }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_write', { + sessionId: 'pty_missing', + data: 'ls\r', + approvalSessionId: 'approval_1', + }) + + expect(result.isError).toBe(true) + expect((result.structuredContent as Record).operation).toBe('pty_write') + }) + }) + + // ----------------------------------------------------------------------- + // Full lifecycle: create → send_input → read → resize → destroy + // ----------------------------------------------------------------------- + + describe('full PTY lifecycle', () => { + it('create → send_input → read → resize → destroy', async () => { + runtime.config = createTestConfig({ approvalMode: 'actions' }) + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 7000, + }) + vi.mocked(readPtyScreen).mockReturnValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '$ ls\nfile.txt', + pid: 7000, + }) + vi.mocked(destroyPtySession).mockReturnValue(true) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + // Create + const createResult = await executeApprovedPtyCreate(runtime, { + rows: 24, + cols: 80, + cwd: '/tmp', + stepId: 'step_life', + approvalSessionId: 'approval_1', + }) + expect((createResult.structuredContent as Record).status).toBe('ok') + expect(runtime.stateManager.getActivePtyGrants()).toHaveLength(1) + + // Send input + await invoke('pty_send_input', { sessionId: 'pty_1', data: 'ls\r', approvalSessionId: 'approval_1' }) + expect(writeToPty).toHaveBeenCalledWith('pty_1', { data: 'ls\r' }) + + // Read screen + const readResult = await invoke('pty_read_screen', { sessionId: 'pty_1', approvalSessionId: 'approval_1' }) + expect((readResult.structuredContent as Record).screenContent).toBe('$ ls\nfile.txt') + + // Resize + await invoke('pty_resize', { sessionId: 'pty_1', rows: 48, cols: 160, approvalSessionId: 'approval_1' }) + expect(resizePty).toHaveBeenCalledWith('pty_1', { cols: 160, rows: 48 }) + + // Destroy + await invoke('pty_destroy', { sessionId: 'pty_1', approvalSessionId: 'approval_1' }) + expect(runtime.stateManager.getActivePtyGrants()).toHaveLength(0) + expect(runtime.stateManager.getPtySessions()).toHaveLength(0) + + // Audit log has all 5 events + const auditLog = runtime.stateManager.getPtyAuditLog() + const events = auditLog.map(e => e.event) + expect(events).toEqual(['create', 'send_input', 'read_screen', 'resize', 'destroy']) + }) + + it('rejects PTY operations without an active grant when approvals are enabled', async () => { + runtime.config = createTestConfig({ approvalMode: 'actions' }) + const { server, invoke } = createMockServer() + registerPtyTools({ server, runtime }) + + const sendResult = await invoke('pty_send_input', { + sessionId: 'pty_missing', + data: 'ls\r', + approvalSessionId: 'approval_1', + }) + + expect(sendResult.isError).toBe(true) + expect((sendResult.structuredContent as Record).status).toBe('pty_grant_required') + }) + }) +}) diff --git a/services/computer-use-mcp/src/server/register-pty.test.ts b/services/computer-use-mcp/src/server/register-pty.test.ts new file mode 100644 index 000000000..c69e7c193 --- /dev/null +++ b/services/computer-use-mcp/src/server/register-pty.test.ts @@ -0,0 +1,193 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { RunStateManager } from '../state' +import { + createPtySession, + isPtyAvailable, + listPtySessions, + readPtyScreen, +} from '../terminal/pty-runner' +import { createTestConfig } from '../test-fixtures' +import { registerPtyTools } from './register-pty' + +vi.mock('../terminal/pty-runner', () => ({ + createPtySession: vi.fn(), + destroyAllPtySessions: vi.fn(), + destroyPtySession: vi.fn(), + getPtyAvailabilityInfo: vi.fn().mockResolvedValue({ available: true }), + isPtyAvailable: vi.fn(), + listPtySessions: vi.fn(), + readPtyScreen: vi.fn(), + resizePty: vi.fn(), + writeToPty: vi.fn(), +})) + +type ToolHandler = (args: Record) => Promise + +function createMockServer() { + const handlers = new Map() + + return { + server: { + tool(name: string, _schema: unknown, handler: ToolHandler) { + handlers.set(name, handler) + }, + } as unknown as McpServer, + async invoke(name: string, args: Record = {}) { + const handler = handlers.get(name) + if (!handler) { + throw new Error(`Missing registered tool: ${name}`) + } + + return await handler(args) + }, + } +} + +describe('registerPtyTools', () => { + let runtime: ComputerUseServerRuntime + let pendingActions: Array> + + beforeEach(() => { + pendingActions = [] + runtime = { + config: createTestConfig({ approvalMode: 'never' }), + stateManager: new RunStateManager(), + session: { + createPendingAction: vi.fn((record: Record) => { + const pending = { ...record, id: `pending_${pendingActions.length + 1}`, createdAt: new Date().toISOString() } + pendingActions.push(pending) + return pending + }), + listPendingActions: vi.fn(() => pendingActions), + record: vi.fn().mockResolvedValue(undefined), + }, + } as unknown as ComputerUseServerRuntime + vi.clearAllMocks() + }) + + it('creates PTY sessions, tracks lifecycle, and binds them to workflow steps', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_1', + alive: true, + rows: 30, + cols: 120, + screenContent: '', + pid: 4242, + }) + const { server, invoke } = createMockServer() + + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_create', { + rows: 30, + cols: 120, + cwd: '/tmp/project', + workflowStepLabel: 'Run TUI check', + }) + + expect(createPtySession).toHaveBeenCalledWith(runtime.config, { + rows: 30, + cols: 120, + cwd: '/tmp/project', + }) + expect((result.structuredContent as Record).status).toBe('ok') + expect(runtime.stateManager.getState().activePtySessionId).toBe('pty_1') + expect(runtime.stateManager.getPtySessions()).toEqual([ + expect.objectContaining({ + id: 'pty_1', + alive: true, + rows: 30, + cols: 120, + pid: 4242, + cwd: '/tmp/project', + boundWorkflowStepLabel: 'Run TUI check', + }), + ]) + }) + + it('reports tracked PTY metadata in pty_get_status', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(listPtySessions).mockReturnValue([ + { + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 9001, + }, + ]) + runtime.stateManager.registerPtySession({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + pid: 9001, + cwd: '/tmp/project', + }) + runtime.stateManager.bindPtySessionToStep('pty_1', 'Inspect terminal') + runtime.stateManager.touchPtySession('pty_1') + const { server, invoke } = createMockServer() + + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_get_status') + const structured = result.structuredContent as Record + + expect(structured.ptyAvailable).toBe(true) + expect(structured.sessions).toEqual([ + expect.objectContaining({ + id: 'pty_1', + alive: true, + pid: 9001, + rows: 24, + cols: 80, + boundWorkflowStepLabel: 'Inspect terminal', + lastInteractionAt: expect.any(String), + }), + ]) + }) + + it('reads screen content from tracked sessions and updates session liveness', async () => { + runtime.stateManager.registerPtySession({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + pid: 9001, + cwd: '/tmp/project', + }) + vi.mocked(readPtyScreen).mockReturnValue({ + id: 'pty_1', + alive: false, + rows: 24, + cols: 80, + screenContent: 'htop', + pid: 9001, + }) + const { server, invoke } = createMockServer() + + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_read_screen', { + sessionId: 'pty_1', + maxLines: 24, + }) + + expect(readPtyScreen).toHaveBeenCalledWith('pty_1', { maxLines: 24 }) + expect((result.structuredContent as Record).screenContent).toBe('htop') + expect(runtime.stateManager.getPtySessions()).toEqual([ + expect.objectContaining({ + id: 'pty_1', + alive: false, + lastInteractionAt: expect.any(String), + }), + ]) + }) +}) diff --git a/services/computer-use-mcp/src/server/register-pty.ts b/services/computer-use-mcp/src/server/register-pty.ts new file mode 100644 index 000000000..547afbafe --- /dev/null +++ b/services/computer-use-mcp/src/server/register-pty.ts @@ -0,0 +1,664 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' +import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js' + +import type { + ForegroundContext, + PolicyDecision, + PtyCreateApprovalInput, +} from '../types' +import type { ComputerUseServerRuntime } from './runtime' + +import process from 'node:process' + +import { z } from 'zod' + +import { + createPtySession, + destroyAllPtySessions, + destroyPtySession, + getPtyAvailabilityInfo, + listPtySessions, + readPtyScreen, + resizePty, + writeToPty, +} from '../terminal/pty-runner' +import { textContent } from './content' +import { buildApprovalResponse } from './responses' + +export interface RegisterPtyToolsOptions { + server: McpServer + runtime: ComputerUseServerRuntime +} + +/** + * Truncate to a safe audit preview — never log full sensitive input. + * Returns at most `maxLen` characters followed by ellipsis if truncated. + */ +function auditPreview(data: string, maxLen = 80): string { + if (data.length <= maxLen) + return data + return `${data.slice(0, maxLen)}…` +} + +function requiresPtyApproval(runtime: ComputerUseServerRuntime) { + return runtime.config.approvalMode !== 'never' +} + +function getApprovalContext(runtime: ComputerUseServerRuntime): ForegroundContext { + return runtime.stateManager.getState().foregroundContext + ?? { available: false, platform: process.platform as NodeJS.Platform } +} + +function buildPtyApprovalDecision(): PolicyDecision { + return { + allowed: true, + requiresApproval: true, + reason: 'Creating an interactive PTY session requires approval.', + reasons: ['Creating an interactive PTY session requires approval.'], + riskLevel: 'high', + estimatedOperationUnits: 4, + } +} + +function buildApprovalSessionRequiredResponse(operation: string): CallToolResult { + return { + isError: true, + content: [ + textContent(`${operation} requires an approval session id when approvals are enabled.`), + ], + structuredContent: { + status: 'approval_session_required', + operation, + }, + } +} + +function buildPtyGrantRequiredResponse(operation: string, sessionId: string): CallToolResult { + return { + isError: true, + content: [ + textContent(`${operation} requires an active PTY Open Grant for session ${sessionId}. Create or approve the PTY session first.`), + ], + structuredContent: { + status: 'pty_grant_required', + operation, + sessionId, + }, + } +} + +function requirePtyGrant(params: { + runtime: ComputerUseServerRuntime + operation: string + sessionId: string + approvalSessionId?: string +}): CallToolResult | undefined { + if (!requiresPtyApproval(params.runtime)) + return undefined + + if (!params.approvalSessionId) { + return buildApprovalSessionRequiredResponse(params.operation) + } + + if (!params.runtime.stateManager.hasPtyApprovalGrant(params.approvalSessionId, params.sessionId)) { + return buildPtyGrantRequiredResponse(params.operation, params.sessionId) + } + + return undefined +} + +export async function executeApprovedPtyCreate( + runtime: ComputerUseServerRuntime, + { + rows, + cols, + cwd, + stepId, + workflowStepLabel, + approvalSessionId, + }: PtyCreateApprovalInput, +): Promise { + const availability = await getPtyAvailabilityInfo() + if (!availability.available) { + return { + isError: true, + content: [textContent(`PTY support unavailable: ${availability.error || 'node-pty could not be loaded.'}`)], + structuredContent: { + status: 'unavailable', + ...(availability.error ? { error: availability.error } : {}), + }, + } + } + + try { + const session = await createPtySession(runtime.config, { rows, cols, cwd }) + + runtime.stateManager.registerPtySession({ + id: session.id, + alive: session.alive, + rows: session.rows, + cols: session.cols, + pid: session.pid, + cwd, + }) + if (stepId) { + runtime.stateManager.bindPtySessionToStepId(session.id, stepId) + } + if (workflowStepLabel) { + runtime.stateManager.bindPtySessionToStep(session.id, workflowStepLabel) + } + + if (requiresPtyApproval(runtime) && approvalSessionId) { + runtime.stateManager.grantPtyApproval(approvalSessionId, session.id) + } + + const task = runtime.stateManager.getState().activeTask + const currentStep = task?.steps[task.currentStepIndex] + runtime.stateManager.appendPtyAudit({ + taskId: task?.id, + stepId: currentStep?.stepId, + ptySessionId: session.id, + event: 'create', + cwd, + rows: session.rows, + cols: session.cols, + pid: session.pid, + }) + + return { + content: [ + textContent(`PTY session created: ${session.id} (${session.cols}x${session.rows}, pid ${session.pid}).`), + ], + structuredContent: { + status: 'ok', + session: { + id: session.id, + pid: session.pid, + rows: session.rows, + cols: session.cols, + alive: session.alive, + stepId, + workflowStepLabel, + }, + ...(requiresPtyApproval(runtime) && approvalSessionId + ? { + grantScope: 'pty_session', + approvalSessionId, + } + : {}), + }, + } + } + catch (error) { + const message = error instanceof Error ? error.message : String(error) + + await runtime.session.record({ + event: 'failed', + toolName: 'pty_create', + action: { + kind: 'pty_create', + input: { + rows, + cols, + cwd, + stepId, + workflowStepLabel, + approvalSessionId, + }, + }, + context: getApprovalContext(runtime), + policy: buildPtyApprovalDecision(), + result: { error: message }, + }) + + return { + isError: true, + content: [textContent(`PTY create failed: ${message}`)], + structuredContent: { + status: 'error', + error: message, + }, + } + } +} + +export function registerPtyTools({ server, runtime }: RegisterPtyToolsOptions) { + // Helper: resolve current task/step ids for audit + function currentIds(): { taskId?: string, stepId?: string } { + const task = runtime.stateManager.getState().activeTask + if (!task) + return {} + const step = task.steps[task.currentStepIndex] + return { taskId: task.id, stepId: step?.stepId } + } + + server.tool( + 'pty_get_status', + {}, + async () => { + const availability = await getPtyAvailabilityInfo() + const sessions = availability.available ? listPtySessions() : [] + const trackedSessions = runtime.stateManager.getPtySessions() + + return { + content: [ + textContent(`PTY support: ${availability.available ? 'available' : `unavailable (${availability.error || 'node-pty could not be loaded'})`}. Active sessions: ${sessions.length}.`), + ], + structuredContent: { + status: 'ok', + ptyAvailable: availability.available, + ...(availability.error ? { error: availability.error } : {}), + sessions: sessions.map(s => ({ + id: s.id, + alive: s.alive, + pid: s.pid, + rows: s.rows, + cols: s.cols, + boundStepId: trackedSessions.find(entry => entry.id === s.id)?.boundStepId, + boundWorkflowStepLabel: trackedSessions.find(entry => entry.id === s.id)?.boundWorkflowStepLabel, + lastInteractionAt: trackedSessions.find(entry => entry.id === s.id)?.lastInteractionAt, + })), + }, + } + }, + ) + + server.tool( + 'pty_create', + { + rows: z.number().int().min(1).max(200).optional().describe('Terminal rows (default: 24)'), + cols: z.number().int().min(1).max(500).optional().describe('Terminal columns (default: 80)'), + cwd: z.string().optional().describe('Initial working directory'), + stepId: z.string().min(1).optional().describe('Stable workflow step id to bind this PTY session to'), + workflowStepLabel: z.string().min(1).optional().describe('(deprecated) Workflow step label for backward compat'), + approvalSessionId: z.string().min(1).optional().describe('(internal) Approval session id used to bind the PTY Open Grant'), + }, + async ({ rows, cols, cwd, stepId, workflowStepLabel, approvalSessionId }) => { + const availability = await getPtyAvailabilityInfo() + if (!availability.available) { + return { + isError: true, + content: [textContent(`PTY support unavailable: ${availability.error || 'node-pty could not be loaded.'}`)], + structuredContent: { + status: 'unavailable', + ...(availability.error ? { error: availability.error } : {}), + }, + } + } + + if (requiresPtyApproval(runtime)) { + if (!approvalSessionId) { + return buildApprovalSessionRequiredResponse('pty_create') + } + + const decision = buildPtyApprovalDecision() + const context = getApprovalContext(runtime) + const pending = runtime.session.createPendingAction({ + toolName: 'pty_create', + action: { + kind: 'pty_create', + input: { + rows, + cols, + cwd, + stepId, + workflowStepLabel, + approvalSessionId, + }, + }, + policy: decision, + context, + }) + runtime.stateManager.setPendingApprovalCount(runtime.session.listPendingActions().length) + + await runtime.session.record({ + event: 'approval_required', + toolName: 'pty_create', + action: pending.action, + context, + policy: decision, + result: { + pendingActionId: pending.id, + }, + }) + + return buildApprovalResponse(pending, decision, context, { + intent: 'Create an interactive PTY session', + approvalReason: 'PTY session creation opens an interactive terminal surface and should be explicitly approved once per session.', + }) + } + + return executeApprovedPtyCreate(runtime, { + rows, + cols, + cwd, + stepId, + workflowStepLabel, + approvalSessionId, + }) + }, + ) + + const createSendInputHandler = (toolName: 'pty_send_input' | 'pty_write') => async ({ sessionId, data, approvalSessionId }: { sessionId: string, data: string, approvalSessionId?: string }) => { + const grantError = requirePtyGrant({ + runtime, + operation: toolName, + sessionId, + approvalSessionId, + }) + if (grantError) { + return grantError + } + + try { + writeToPty(sessionId, { data }) + runtime.stateManager.touchPtySession(sessionId) + + // Audit: only log byte count + truncated preview, never full content + const ids = currentIds() + runtime.stateManager.appendPtyAudit({ + ...ids, + ptySessionId: sessionId, + event: 'send_input', + byteCount: data.length, + inputPreview: auditPreview(data), + }) + + return { + content: [textContent(`Wrote ${data.length} byte(s) to ${sessionId}.`)], + structuredContent: { + status: 'ok', + sessionId, + bytesWritten: data.length, + }, + } + } + catch (error) { + return { + isError: true, + content: [textContent(`PTY send_input failed: ${error instanceof Error ? error.message : String(error)}`)], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + } + + const sendInputSchema = { + sessionId: z.string().min(1).describe('PTY session id from pty_create'), + data: z.string().describe('Data to write to the PTY (keystrokes, commands, etc.). Use \\r for Enter, \\x03 for Ctrl+C.'), + approvalSessionId: z.string().min(1).optional().describe('(internal) Approval session id used to validate the PTY Open Grant'), + } + + // Primary name + server.tool('pty_send_input', sendInputSchema, createSendInputHandler('pty_send_input')) + // Compat alias — kept for backward compatibility, not the canonical name + server.tool('pty_write', sendInputSchema, createSendInputHandler('pty_write')) + + server.tool( + 'pty_read_screen', + { + sessionId: z.string().min(1).describe('PTY session id'), + maxLines: z.number().int().min(1).max(500).optional().describe('Maximum lines to return from the terminal buffer (default: terminal rows)'), + approvalSessionId: z.string().min(1).optional().describe('(internal) Approval session id used to validate the PTY Open Grant'), + }, + async ({ sessionId, maxLines, approvalSessionId }) => { + const grantError = requirePtyGrant({ + runtime, + operation: 'pty_read_screen', + sessionId, + approvalSessionId, + }) + if (grantError) { + return grantError + } + + try { + const session = readPtyScreen(sessionId, { maxLines }) + runtime.stateManager.touchPtySession(sessionId) + runtime.stateManager.updatePtySessionAlive(sessionId, session.alive) + + // Audit + const ids = currentIds() + const lineCount = session.screenContent ? session.screenContent.split('\n').length : 0 + runtime.stateManager.appendPtyAudit({ + ...ids, + ptySessionId: sessionId, + event: 'read_screen', + returnedLineCount: lineCount, + alive: session.alive, + }) + + return { + content: [textContent(session.screenContent || '(empty)')], + structuredContent: { + status: 'ok', + session: { + id: session.id, + alive: session.alive, + pid: session.pid, + rows: session.rows, + cols: session.cols, + }, + sessionId: session.id, + alive: session.alive, + rows: session.rows, + cols: session.cols, + screenContent: session.screenContent, + }, + } + } + catch (error) { + return { + isError: true, + content: [textContent(`PTY read failed: ${error instanceof Error ? error.message : String(error)}`)], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'pty_resize', + { + sessionId: z.string().min(1).describe('PTY session id'), + cols: z.number().int().min(1).max(500).describe('New terminal column count'), + rows: z.number().int().min(1).max(200).describe('New terminal row count'), + approvalSessionId: z.string().min(1).optional().describe('(internal) Approval session id used to validate the PTY Open Grant'), + }, + async ({ sessionId, cols, rows, approvalSessionId }) => { + const grantError = requirePtyGrant({ + runtime, + operation: 'pty_resize', + sessionId, + approvalSessionId, + }) + if (grantError) { + return grantError + } + + try { + resizePty(sessionId, { cols, rows }) + + // Audit + const ids = currentIds() + runtime.stateManager.appendPtyAudit({ + ...ids, + ptySessionId: sessionId, + event: 'resize', + rows, + cols, + }) + + return { + content: [textContent(`Resized ${sessionId} to ${cols}x${rows}.`)], + structuredContent: { + status: 'ok', + sessionId, + cols, + rows, + }, + } + } + catch (error) { + return { + isError: true, + content: [textContent(`PTY resize failed: ${error instanceof Error ? error.message : String(error)}`)], + structuredContent: { + status: 'error', + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'pty_destroy', + { + sessionId: z.string().min(1).describe('PTY session id to destroy'), + approvalSessionId: z.string().min(1).optional().describe('(internal) Approval session id used to validate the PTY Open Grant'), + }, + async ({ sessionId, approvalSessionId }) => { + const grantError = requirePtyGrant({ + runtime, + operation: 'pty_destroy', + sessionId, + approvalSessionId, + }) + if (grantError) { + return grantError + } + + const destroyed = destroyPtySession(sessionId) + if (destroyed) { + // Revoke the Open Grant for this session + runtime.stateManager.revokePtyApproval(sessionId) + runtime.stateManager.unregisterPtySession(sessionId) + + // Audit + const ids = currentIds() + runtime.stateManager.appendPtyAudit({ + ...ids, + ptySessionId: sessionId, + event: 'destroy', + actor: 'tool_call', + outcome: 'ok', + }) + } + + return { + content: [textContent(destroyed ? `Destroyed ${sessionId}.` : `Session not found: ${sessionId}.`)], + structuredContent: { + status: destroyed ? 'ok' : 'not_found', + sessionId, + }, + } + }, + ) +} + +// --------------------------------------------------------------------------- +// Workflow self-acquire PTY callback factory +// --------------------------------------------------------------------------- + +/** + * Creates an `AcquirePtyForStep` callback that the workflow engine invokes + * when surface resolution determines a step needs a PTY. + * + * The callback goes through the **same** approval / grant / audit pipeline + * as the external `pty_create` MCP tool — no shortcuts. + * + * When `autoApprove` is true the engine is allowed to create the PTY + * directly (mirroring `approvalMode === 'never'`). When false and + * approvals are required, the callback returns `approvalPending: true` so + * the engine can suspend at `before_pty_acquire`. + */ +export function createAcquirePtyCallback( + runtime: ComputerUseServerRuntime, +): import('../workflows/engine').AcquirePtyForStep { + return async ({ taskId, stepId, cwd, rows, cols, autoApprove }) => { + const availability = await getPtyAvailabilityInfo() + if (!availability.available) { + return { + acquired: false, + error: `PTY support unavailable: ${availability.error || 'node-pty could not be loaded.'}`, + } + } + + // When approvals are active AND the caller did not opt into auto-approve, + // we must go through the pending-action → user-approval flow. + if (requiresPtyApproval(runtime) && !autoApprove) { + const { randomUUID } = await import('node:crypto') + const approvalSessionId = randomUUID() + const decision = buildPtyApprovalDecision() + const context = getApprovalContext(runtime) + + runtime.session.createPendingAction({ + toolName: 'pty_create', + action: { + kind: 'pty_create', + input: { rows, cols, cwd, stepId, approvalSessionId } satisfies PtyCreateApprovalInput, + }, + policy: decision, + context, + }) + runtime.stateManager.setPendingApprovalCount( + runtime.session.listPendingActions().length, + ) + + await runtime.session.record({ + event: 'approval_required', + toolName: 'pty_create', + action: { + kind: 'pty_create', + input: { rows, cols, cwd, stepId }, + }, + context, + policy: decision, + result: { workflow_self_acquire: true, taskId }, + }) + + return { acquired: false, approvalPending: true } + } + + // Auto-approve path (or approval mode is "never") — create directly. + // Generate an approvalSessionId so the grant machinery stays consistent. + const { randomUUID } = await import('node:crypto') + const approvalSessionId = randomUUID() + + const result = await executeApprovedPtyCreate(runtime, { + rows, + cols, + cwd, + stepId, + approvalSessionId, + }) + + if (result.isError) { + const msg = result.content?.[0] && 'text' in result.content[0] + ? result.content[0].text + : 'PTY creation failed' + return { acquired: false, error: msg } + } + + const structured = result.structuredContent as Record | undefined + const session = structured?.session as Record | undefined + const ptySessionId = (session?.id ?? '') as string + + if (!ptySessionId) { + return { acquired: false, error: 'PTY created but session id missing from result.' } + } + + return { acquired: true, ptySessionId } + } +} + +/** + * Cleanup helper — destroy all PTY sessions. Called on server shutdown. + */ +export { destroyAllPtySessions } diff --git a/services/computer-use-mcp/src/server/register-task-memory.ts b/services/computer-use-mcp/src/server/register-task-memory.ts new file mode 100644 index 000000000..1e5084f0f --- /dev/null +++ b/services/computer-use-mcp/src/server/register-task-memory.ts @@ -0,0 +1,122 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { z } from 'zod' + +import { textContent } from './content' + +/** + * Register task memory MCP tools on the server. + * + * - `task_memory_update`: LLM calls this to write/merge task execution state. + * - `task_memory_get`: LLM calls this to read the current task memory snapshot. + * - `task_memory_clear`: LLM calls this to reset task memory. + */ +export function registerTaskMemoryTools(server: McpServer, runtime: ComputerUseServerRuntime) { + // ------------------------------------------------------------------ + // task_memory_update + // ------------------------------------------------------------------ + server.tool( + 'task_memory_update', + { + status: z.enum(['active', 'blocked', 'done']).optional().describe('Task status'), + goal: z.string().optional().describe('High-level task goal'), + currentStep: z.string().optional().describe('What is being done right now'), + confirmedFacts: z.array(z.string()).optional().describe('Facts confirmed with evidence'), + artifacts: z.array(z.object({ + label: z.string().describe('Short human-readable label'), + value: z.string().describe('Artifact value (path, URL, or short text)'), + kind: z.enum(['file', 'url', 'tool', 'note']).describe('Artifact category'), + })).optional().describe('Key artifacts produced'), + blockers: z.array(z.string()).optional().describe('What is blocking progress'), + nextStep: z.string().optional().describe('What to do next'), + plan: z.array(z.string()).optional().describe('Ordered plan steps'), + workingAssumptions: z.array(z.string()).optional().describe('Unconfirmed assumptions'), + recentFailureReason: z.string().optional().describe('Why the last attempt failed'), + completionCriteria: z.array(z.string()).optional().describe('How to know when done'), + newTask: z.boolean().optional().describe('Set true ONLY if starting a clearly different task/goal'), + sourceTurnId: z.string().min(1).describe('Stable identifier of the completed turn producing this update'), + sourceTurnIndex: z.number().int().nonnegative().describe('Monotonic index of the completed turn within the session'), + }, + async (params, _extra) => { + const result = runtime.taskMemory.update( + { + status: params.status, + goal: params.goal, + currentStep: params.currentStep, + confirmedFacts: params.confirmedFacts, + artifacts: params.artifacts, + blockers: params.blockers, + nextStep: params.nextStep, + plan: params.plan, + workingAssumptions: params.workingAssumptions, + recentFailureReason: params.recentFailureReason, + completionCriteria: params.completionCriteria, + newTask: params.newTask, + }, + { + sourceTurnId: params.sourceTurnId, + sourceTurnIndex: params.sourceTurnIndex, + }, + ) + + if (result.status === 'updated') { + const { taskMemory: merged } = result + // Sync to run state so other tools can see it via summarizeRunState + runtime.stateManager.updateTaskMemory(merged) + + return { + content: [textContent(`Task memory updated [${merged.status}]${merged.goal ? `: ${merged.goal}` : ''}`)], + structuredContent: { ...merged } as Record, + } + } + + if (result.status === 'ignored-stale') { + return { + content: [textContent(`Task memory update ignored as stale. Latest observed turn is ${result.latestSourceTurnId ?? 'unknown'}${result.latestSourceTurnIndex !== undefined ? ` (#${result.latestSourceTurnIndex})` : ''}.`)], + } + } + + return { + content: [textContent('Task memory update had no meaningful content to persist.')], + } + }, + ) + + // ------------------------------------------------------------------ + // task_memory_get + // ------------------------------------------------------------------ + server.tool( + 'task_memory_get', + {}, + async (_params, _extra) => { + const tm = runtime.taskMemory.get() + if (!tm) { + return { + content: [textContent('No active task memory. Use task_memory_update to start tracking a task.')], + } + } + + return { + content: [textContent(runtime.taskMemory.toContextString())], + structuredContent: { ...tm } as Record, + } + }, + ) + + // ------------------------------------------------------------------ + // task_memory_clear + // ------------------------------------------------------------------ + server.tool( + 'task_memory_clear', + {}, + async (_params, _extra) => { + runtime.taskMemory.clear() + runtime.stateManager.clearTaskMemory() + return { + content: [textContent('Task memory cleared.')], + } + }, + ) +} diff --git a/services/computer-use-mcp/src/server/register-tools-pty-approval.test.ts b/services/computer-use-mcp/src/server/register-tools-pty-approval.test.ts new file mode 100644 index 000000000..3b23476df --- /dev/null +++ b/services/computer-use-mcp/src/server/register-tools-pty-approval.test.ts @@ -0,0 +1,175 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { RunStateManager } from '../state' +import { createPtySession, isPtyAvailable } from '../terminal/pty-runner' +import { createTestConfig } from '../test-fixtures' +import { registerComputerUseTools } from './register-tools' + +vi.mock('../terminal/pty-runner', () => ({ + createPtySession: vi.fn(), + destroyAllPtySessions: vi.fn(), + destroyPtySession: vi.fn(), + getPtyAvailabilityInfo: vi.fn().mockResolvedValue({ available: true }), + isPtyAvailable: vi.fn(), + listPtySessions: vi.fn(), + readPtyScreen: vi.fn(), + resizePty: vi.fn(), + writeToPty: vi.fn(), +})) + +type ToolHandler = (args: Record) => Promise + +function createMockServer() { + const handlers = new Map() + + return { + server: { + tool(name: string, _schema: unknown, handler: ToolHandler) { + handlers.set(name, handler) + }, + } as unknown as McpServer, + async invoke(name: string, args: Record = {}) { + const handler = handlers.get(name) + if (!handler) { + throw new Error(`Missing registered tool: ${name}`) + } + + return await handler(args) + }, + } +} + +describe('registerComputerUseTools: PTY approval bridge', () => { + let runtime: ComputerUseServerRuntime + let pendingActions: Map> + + beforeEach(() => { + pendingActions = new Map() + runtime = { + config: createTestConfig({ approvalMode: 'actions' }), + stateManager: new RunStateManager(), + session: { + createPendingAction: vi.fn(), + getPendingAction: vi.fn((id: string) => pendingActions.get(id)), + listPendingActions: vi.fn(() => [...pendingActions.values()]), + removePendingAction: vi.fn((id: string) => pendingActions.delete(id)), + record: vi.fn().mockResolvedValue(undefined), + getBudgetState: vi.fn(() => ({ operationsExecuted: 0, operationUnitsConsumed: 0 })), + getLastScreenshot: vi.fn(() => undefined), + }, + executor: { + getPermissionInfo: vi.fn().mockResolvedValue({}), + }, + terminalRunner: { + getState: vi.fn(() => ({ effectiveCwd: '/tmp' })), + }, + browserDomBridge: { + triggerEvent: vi.fn(), + getStatus: vi.fn(() => ({ enabled: false, connected: false })), + }, + cdpBridgeManager: { + getAvailability: vi.fn(), + }, + taskMemory: {}, + } as unknown as ComputerUseServerRuntime + vi.clearAllMocks() + }) + + it('executes approved pending pty_create through desktop_approve_pending_action', async () => { + vi.mocked(isPtyAvailable).mockResolvedValue(true) + vi.mocked(createPtySession).mockResolvedValue({ + id: 'pty_approved', + alive: true, + rows: 24, + cols: 80, + screenContent: '', + pid: 4321, + }) + + pendingActions.set('pending-pty-1', { + id: 'pending-pty-1', + createdAt: new Date().toISOString(), + toolName: 'pty_create', + action: { + kind: 'pty_create', + input: { + rows: 24, + cols: 80, + cwd: '/tmp/project', + approvalSessionId: 'approval_1', + }, + }, + policy: { + allowed: true, + requiresApproval: true, + reasons: ['Creating an interactive PTY session requires approval.'], + riskLevel: 'high', + estimatedOperationUnits: 4, + }, + context: { + available: false, + platform: 'darwin', + }, + }) + + const { server, invoke } = createMockServer() + registerComputerUseTools({ + server, + runtime, + executeAction: vi.fn(), + enableTestTools: false, + }) + + const result = await invoke('desktop_approve_pending_action', { id: 'pending-pty-1' }) + + expect((result.structuredContent as Record).status).toBe('ok') + expect(createPtySession).toHaveBeenCalledWith(runtime.config, { + rows: 24, + cols: 80, + cwd: '/tmp/project', + }) + expect(runtime.stateManager.getActivePtyGrants()).toEqual([ + expect.objectContaining({ + approvalSessionId: 'approval_1', + ptySessionId: 'pty_approved', + active: true, + }), + ]) + expect((runtime.session.getPendingAction as any)('pending-pty-1')).toBeUndefined() + }) + + it('returns a structured error when browser_dom_trigger_event receives malformed optsJson', async () => { + ;(runtime.browserDomBridge.getStatus as any).mockReturnValue({ + enabled: true, + connected: true, + host: '127.0.0.1', + port: 8765, + pendingRequests: 0, + }) + + const { server, invoke } = createMockServer() + registerComputerUseTools({ + server, + runtime, + executeAction: vi.fn(), + enableTestTools: false, + }) + + const result = await invoke('browser_dom_trigger_event', { + selector: '#app', + eventName: 'click', + optsJson: '{not-valid-json}', + }) + + expect(result.isError).toBe(true) + expect(result.structuredContent).toMatchObject({ + status: 'invalid_params', + field: 'optsJson', + }) + expect((runtime.browserDomBridge.triggerEvent as any)).not.toHaveBeenCalled() + }) +}) diff --git a/services/computer-use-mcp/src/server/register-tools.ts b/services/computer-use-mcp/src/server/register-tools.ts new file mode 100644 index 000000000..6a2bb5e1c --- /dev/null +++ b/services/computer-use-mcp/src/server/register-tools.ts @@ -0,0 +1,1229 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { + BrowserDomFrameResult, + ClickActionInput, + FocusAppActionInput, + OpenAppActionInput, + SecretReadEnvValueActionInput, + TerminalExecActionInput, + TypeTextActionInput, +} from '../types' +import type { WorkflowSuspension } from '../workflows' +import type { ExecuteAction } from './action-executor' +import type { ComputerUseServerRuntime } from './runtime' + +import { z } from 'zod' + +import { getRuntimePreflight } from '../preflight' +import { summarizeRunState } from '../transparency' +import { + createAppBrowseAndActWorkflow, + createDevInspectFailureWorkflow, + createDevOpenWorkspaceWorkflow, + createDevRunTestsWorkflow, + createDevValidateWorkspaceWorkflow, + executeWorkflow, + resumeWorkflow, +} from '../workflows' +import { getBrowserAgentLaunchContext, runBrowserAgentTask } from './browser-agent' +import { textContent } from './content' +import { + describeExecutionTarget, + describeForegroundContext, + summarizeCoordinateSpace, +} from './formatters' +import { refreshRuntimeRunState } from './refresh-run-state' +import { createAcquirePtyCallback, executeApprovedPtyCreate } from './register-pty' +import { formatWorkflowStructuredContent } from './workflow-formatter' +import { createWorkflowPrepToolExecutor } from './workflow-prep-tools' + +export interface RegisterComputerUseToolsOptions { + server: McpServer + runtime: ComputerUseServerRuntime + executeAction: ExecuteAction + enableTestTools: boolean +} + +const optionalTabIdSchema = z.number().int().min(0).optional().describe('Optional browser tab id override; defaults to the active tab') +const optionalFrameIdsSchema = z.array(z.number().int().min(0)).min(1).optional().describe('Optional frame ids to target; omit to let the bridge inspect all frames') + +function toBrowserDomRecord(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) + return undefined + + return value as Record +} + +function unwrapBrowserDomResult(value: unknown) { + const record = toBrowserDomRecord(value) + if (!record) + return value + + if ('data' in record) + return record.data + + return value +} + +function didBrowserDomFrameSucceed(frame: BrowserDomFrameResult) { + const record = toBrowserDomRecord(frame.result) + if (!record) + return Boolean(frame.result) + + if ('success' in record) + return Boolean(record.success) + + return true +} + +function summarizeBrowserDomFrameResults(label: string, results: Array>) { + const successfulFrames = results.filter(didBrowserDomFrameSucceed) + return `${label}: ${successfulFrames.length}/${results.length} frame(s) succeeded.` +} + +function buildBrowserDomUnavailableResponse(runtime: ComputerUseServerRuntime) { + const status = runtime.browserDomBridge.getStatus() + return { + isError: true, + content: [ + textContent(`Browser DOM bridge is unavailable: ${status.lastError || 'the browser extension is not connected yet'}.`), + ], + structuredContent: { + status: 'unavailable', + bridge: status, + }, + } +} + +export function registerComputerUseTools(params: RegisterComputerUseToolsOptions) { + const { server, runtime, executeAction, enableTestTools } = params + const executePrepTool = createWorkflowPrepToolExecutor(runtime) + const acquirePty = createAcquirePtyCallback(runtime) + + async function refreshWorkflowRunState() { + await refreshRuntimeRunState(runtime) + } + + // Workflow suspension state — stored in this closure so that + // workflow_resume and the approve handler can access it. + let suspendedWorkflow: WorkflowSuspension | undefined + + server.tool( + 'desktop_get_capabilities', + {}, + async () => { + const [{ executionTarget, context, displayInfo, browserSurfaceAvailability }, permissionInfo] = await Promise.all([ + refreshRuntimeRunState(runtime), + runtime.executor.getPermissionInfo(), + ]) + const snapshot = runtime.session.getSnapshot() + const preflight = getRuntimePreflight({ + config: runtime.config, + lastScreenshot: runtime.session.getLastScreenshot(), + displayInfo, + executionTarget, + }) + + return { + content: [ + textContent( + `Executor=${runtime.config.executor}, host=${preflight.launchContext.hostName}, target=${describeExecutionTarget(executionTarget)}, sessionTag=${preflight.launchContext.sessionTag || 'missing'}, coordinateSpace=${summarizeCoordinateSpace(preflight.coordinateSpace)}. Foreground=${describeForegroundContext(context)}.`, + ), + ], + structuredContent: { + executor: runtime.executor.describe(), + launchContext: preflight.launchContext, + executionTarget, + displayInfo, + permissions: permissionInfo, + coordinateSpace: preflight.coordinateSpace, + mutationGuards: { + applies: runtime.config.executor !== 'dry-run', + requireSessionTagForMutatingActions: runtime.config.requireSessionTagForMutatingActions, + requireAllowedBoundsForMutatingActions: runtime.config.requireAllowedBoundsForMutatingActions, + requireCoordinateAlignmentForMutatingActions: runtime.config.requireCoordinateAlignmentForMutatingActions, + readyForMutations: preflight.mutationReadinessIssues.length === 0, + blockingIssues: preflight.mutationReadinessIssues, + }, + policy: { + approvalMode: runtime.config.approvalMode, + allowedBounds: runtime.config.allowedBounds, + allowApps: runtime.config.allowApps, + denyApps: runtime.config.denyApps, + denyWindowTitles: runtime.config.denyWindowTitles, + openableApps: runtime.config.openableApps, + maxOperations: runtime.config.maxOperations, + maxOperationUnits: runtime.config.maxOperationUnits, + defaultCaptureAfter: runtime.config.defaultCaptureAfter, + }, + session: snapshot, + foregroundContext: context, + windowAutomation: runtime.config.executor === 'macos-local' + ? 'NSWorkspace + CGWindowList + Quartz' + : runtime.config.executor === 'linux-x11' + ? 'remote X11 runner' + : 'dry-run', + supportedAppsForOpenFocus: runtime.config.openableApps, + approvalUx: 'electron-dialog', + coordScope: 'global-screen', + appPolicy: 'deny-only', + terminalBackend: runtime.terminalRunner.describe().kind, + browserAgent: getBrowserAgentLaunchContext(), + browserDomBridge: runtime.browserDomBridge.getStatus(), + browserSurfaceAvailability, + }, + } + }, + ) + + if (enableTestTools && runtime.executor.openTestTarget) { + server.tool( + 'desktop_open_test_target', + {}, + async () => { + const result = await runtime.executor.openTestTarget!() + + return { + content: [ + textContent(`Opened ${result.appName} on ${describeExecutionTarget(result.executionTarget)}.`), + ], + structuredContent: { + status: 'executed', + appName: result.appName, + windowTitle: result.windowTitle, + recommendedClickPoint: result.recommendedClickPoint, + executionTarget: result.executionTarget, + }, + } + }, + ) + } + + server.tool( + 'desktop_observe_windows', + { + limit: z.number().int().min(1).max(32).optional().describe('Maximum number of visible windows to return'), + app: z.string().optional().describe('Optional app-name substring filter'), + }, + async input => executeAction({ kind: 'observe_windows', input }, 'desktop_observe_windows'), + ) + + server.tool( + 'desktop_screenshot', + { + label: z.string().optional().describe('Optional label for the saved screenshot file'), + }, + async ({ label }) => executeAction({ kind: 'screenshot', input: { label } }, 'desktop_screenshot'), + ) + + server.tool( + 'desktop_open_app', + { + app: z.string().min(1).describe('Application name from COMPUTER_USE_OPENABLE_APPS'), + }, + async (input: OpenAppActionInput) => executeAction({ kind: 'open_app', input }, 'desktop_open_app'), + ) + + server.tool( + 'desktop_focus_app', + { + app: z.string().min(1).describe('Application name from COMPUTER_USE_OPENABLE_APPS'), + }, + async (input: FocusAppActionInput) => executeAction({ kind: 'focus_app', input }, 'desktop_focus_app'), + ) + + server.tool( + 'desktop_click', + { + x: z.number().describe('Absolute screen X coordinate in pixels'), + y: z.number().describe('Absolute screen Y coordinate in pixels'), + button: z.enum(['left', 'right', 'middle']).optional().describe('Mouse button, default left'), + clickCount: z.number().int().min(1).max(2).optional().describe('Number of clicks, default 1'), + captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'), + }, + async (input: ClickActionInput) => executeAction({ kind: 'click', input }, 'desktop_click'), + ) + + server.tool( + 'desktop_type_text', + { + text: z.string().min(1).describe('Text to type into the focused UI element'), + x: z.number().optional().describe('Optional X coordinate to click before typing'), + y: z.number().optional().describe('Optional Y coordinate to click before typing'), + pressEnter: z.boolean().optional().describe('Whether to press Enter after typing'), + captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'), + }, + async (input: TypeTextActionInput) => executeAction({ kind: 'type_text', input }, 'desktop_type_text'), + ) + + server.tool( + 'desktop_press_keys', + { + keys: z.array(z.string()).min(1).describe('Single key chord, e.g. ["ctrl", "l"]'), + captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'), + }, + async input => executeAction({ kind: 'press_keys', input }, 'desktop_press_keys'), + ) + + server.tool( + 'desktop_scroll', + { + x: z.number().optional().describe('Optional X coordinate to move to before scrolling'), + y: z.number().optional().describe('Optional Y coordinate to move to before scrolling'), + deltaX: z.number().optional().describe('Horizontal scroll delta in pixels'), + deltaY: z.number().describe('Vertical scroll delta in pixels'), + captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'), + }, + async input => executeAction({ kind: 'scroll', input }, 'desktop_scroll'), + ) + + server.tool( + 'desktop_wait', + { + durationMs: z.number().int().min(0).max(30_000).describe('Wait time in milliseconds'), + captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the wait'), + }, + async input => executeAction({ kind: 'wait', input }, 'desktop_wait'), + ) + + server.tool( + 'terminal_exec', + { + command: z.string().min(1).describe('Shell command to execute in the local background runner'), + cwd: z.string().optional().describe('Optional working directory override'), + timeoutMs: z.number().int().min(1).max(120_000).optional().describe('Optional timeout override in milliseconds'), + }, + async (input: TerminalExecActionInput) => executeAction({ kind: 'terminal_exec', input }, 'terminal_exec'), + ) + + server.tool( + 'terminal_get_state', + {}, + async () => { + const terminalState = runtime.session.getTerminalState() + return { + content: [ + textContent(`Terminal runner cwd=${terminalState.effectiveCwd}, lastExitCode=${terminalState.lastExitCode ?? 'n/a'}, lastCommand=${terminalState.lastCommandSummary || 'n/a'}.`), + ], + structuredContent: { + status: 'ok', + terminalState, + }, + } + }, + ) + + server.tool( + 'terminal_reset_state', + { + reason: z.string().optional().describe('Optional reset note for the audit log'), + }, + async input => executeAction({ kind: 'terminal_reset', input }, 'terminal_reset_state'), + ) + + server.tool( + 'secret_read_env_value', + { + filePath: z.string().min(1).describe('Absolute or explicit env file path to inspect, for example /Users/liuziheng/airi/.env'), + keys: z.array(z.string().min(1)).min(1).max(16).describe('Candidate env variable names to try in order, e.g. ["AIRI_E2E_DISCORD_TOKEN", "DISCORD_BOT_TOKEN"]'), + allowPlaceholder: z.boolean().optional().describe('Whether to allow obvious placeholder/template values such as replace-with-your-token'), + }, + async (input: SecretReadEnvValueActionInput) => executeAction({ kind: 'secret_read_env_value', input }, 'secret_read_env_value'), + ) + + server.tool( + 'clipboard_read_text', + { + maxLength: z.number().int().min(1).max(32_768).optional().describe('Optional maximum number of characters to return from the clipboard'), + trim: z.boolean().optional().describe('Whether to trim leading/trailing whitespace before returning the text (default: true)'), + }, + async input => executeAction({ kind: 'clipboard_read_text', input }, 'clipboard_read_text'), + ) + + server.tool( + 'clipboard_write_text', + { + text: z.string().describe('Text to place into the system clipboard'), + }, + async input => executeAction({ kind: 'clipboard_write_text', input }, 'clipboard_write_text'), + ) + + server.tool( + 'browser_dom_get_bridge_status', + {}, + async () => { + const bridge = runtime.browserDomBridge.getStatus() + return { + content: [ + textContent(`Browser DOM bridge ${bridge.connected ? 'connected' : 'disconnected'} on ws://${bridge.host}:${bridge.port}.`), + ], + structuredContent: { + status: 'ok', + bridge, + }, + } + }, + ) + + server.tool( + 'browser_agent_get_status', + {}, + async () => { + const launchContext = getBrowserAgentLaunchContext() + return { + content: [ + textContent(`Browser agent root ${launchContext.rootExists ? 'ready' : 'missing'} at ${launchContext.cliCwd}; python=${launchContext.pythonCommand}; cdp=${launchContext.cdpUrl}.`), + ], + structuredContent: { + status: launchContext.rootExists ? 'ok' : 'missing', + browserAgent: launchContext, + }, + } + }, + ) + + server.tool( + 'browser_agent_run', + { + instruction: z.string().min(1).describe('Goal-driven browser instruction for the autonomous browser agent.'), + agent: z.enum(['google', 'kimi']).optional().describe('Browser agent backend to use (default: google).'), + cdpUrl: z.string().optional().describe('Optional Chrome CDP endpoint override, e.g. http://localhost:9222'), + maxTurns: z.number().int().min(1).max(80).optional().describe('Maximum browser-agent reasoning turns (default: 30).'), + timeoutMs: z.number().int().min(1_000).max(900_000).optional().describe('End-to-end timeout for the delegated browser task (default: 180000).'), + }, + async ({ instruction, agent, cdpUrl, maxTurns, timeoutMs }) => { + const launchContext = getBrowserAgentLaunchContext({ cdpUrl }) + + if (!launchContext.rootExists) { + return { + isError: true, + content: [ + textContent(`Browser agent root is missing: ${launchContext.cliCwd}.`), + ], + structuredContent: { + status: 'missing', + browserAgent: launchContext, + }, + } + } + + try { + const result = await runBrowserAgentTask({ + instruction, + agent, + cdpUrl, + maxTurns, + timeoutMs, + }) + + return { + content: [ + textContent(`Browser agent ${result.success ? 'completed' : 'stopped'} on ${result.payload?.url || result.cdpUrl}.`), + ], + structuredContent: { + status: result.success ? 'completed' : 'failed', + browserAgent: { + instruction: result.instruction, + agent: result.agent, + cdpUrl: result.cdpUrl, + cliCwd: result.cliCwd, + cliModule: result.cliModule, + pythonCommand: result.pythonCommand, + exitCode: result.exitCode, + timedOut: result.timedOut, + stderrLines: result.stderrLines, + }, + payload: result.payload, + }, + } + } + catch (error) { + return { + isError: true, + content: [ + textContent(`Browser agent failed: ${error instanceof Error ? error.message : String(error)}`), + ], + structuredContent: { + status: 'error', + browserAgent: launchContext, + error: error instanceof Error ? error.message : String(error), + }, + } + } + }, + ) + + server.tool( + 'browser_dom_get_active_tab', + {}, + async () => { + if (!runtime.browserDomBridge.getStatus().connected) + return buildBrowserDomUnavailableResponse(runtime) + + const activeTab = await runtime.browserDomBridge.getActiveTab() + return { + content: [ + textContent(`Active browser tab: ${String(activeTab?.title || activeTab?.url || 'unknown')}.`), + ], + structuredContent: { + status: 'ok', + activeTab, + bridge: runtime.browserDomBridge.getStatus(), + }, + } + }, + ) + + server.tool( + 'browser_dom_read_page', + { + includeText: z.boolean().optional().describe('Whether to include truncated body text for each frame'), + maxElements: z.number().int().min(1).max(500).optional().describe('Maximum interactive elements per frame to collect'), + tabId: optionalTabIdSchema, + frameIds: optionalFrameIdsSchema, + }, + async ({ includeText, maxElements, tabId, frameIds }) => { + if (!runtime.browserDomBridge.getStatus().connected) + return buildBrowserDomUnavailableResponse(runtime) + + const frames = await runtime.browserDomBridge.readAllFramesDom({ + includeText, + maxElements, + tabId, + frameIds, + }) + const interactiveElementCount = frames.reduce((count, frame) => { + const payload = unwrapBrowserDomResult(frame.result) + const record = toBrowserDomRecord(payload) + const elements = Array.isArray(record?.interactiveElements) ? record.interactiveElements : [] + return count + elements.length + }, 0) + + return { + content: [ + textContent(`Read DOM from ${frames.length} frame(s); collected ${interactiveElementCount} interactive element(s).`), + ], + structuredContent: { + status: 'ok', + frames, + bridge: runtime.browserDomBridge.getStatus(), + }, + } + }, + ) + + server.tool( + 'browser_dom_find_elements', + { + selector: z.string().min(1).describe('CSS selector to query in the active tab frames'), + maxResults: z.number().int().min(1).max(50).optional().describe('Maximum matched elements to include per frame'), + tabId: optionalTabIdSchema, + frameIds: optionalFrameIdsSchema, + }, + async ({ selector, maxResults, tabId, frameIds }) => { + if (!runtime.browserDomBridge.getStatus().connected) + return buildBrowserDomUnavailableResponse(runtime) + + const results = await runtime.browserDomBridge.findElements({ + selector, + maxResults, + tabId, + frameIds, + }) + return { + content: [ + textContent(summarizeBrowserDomFrameResults(`find_elements for "${selector}"`, results)), + ], + structuredContent: { + status: 'ok', + selector, + results, + bridge: runtime.browserDomBridge.getStatus(), + }, + } + }, + ) + + server.tool( + 'browser_dom_click', + { + selector: z.string().min(1).describe('CSS selector to click via the browser extension bridge'), + tabId: optionalTabIdSchema, + frameIds: optionalFrameIdsSchema, + }, + async ({ selector, tabId, frameIds }) => { + if (!runtime.browserDomBridge.getStatus().connected) + return buildBrowserDomUnavailableResponse(runtime) + + const result = await runtime.browserDomBridge.clickSelector({ + selector, + tabId, + frameIds, + }) + return { + content: [ + textContent(`Clicked selector "${selector}" in frame ${result.targetFrameId} at (${result.targetPoint.x}, ${result.targetPoint.y}).`), + ], + structuredContent: { + status: 'ok', + selector, + ...result, + bridge: runtime.browserDomBridge.getStatus(), + }, + } + }, + ) + + server.tool( + 'browser_dom_read_input_value', + { + selector: z.string().min(1).describe('CSS selector for the input/select/textarea element'), + tabId: optionalTabIdSchema, + frameIds: optionalFrameIdsSchema, + }, + async ({ selector, tabId, frameIds }) => { + if (!runtime.browserDomBridge.getStatus().connected) + return buildBrowserDomUnavailableResponse(runtime) + + const results = await runtime.browserDomBridge.readInputValue({ + selector, + tabId, + frameIds, + }) + return { + content: [ + textContent(summarizeBrowserDomFrameResults(`read_input_value for "${selector}"`, results)), + ], + structuredContent: { + status: 'ok', + selector, + results, + bridge: runtime.browserDomBridge.getStatus(), + }, + } + }, + ) + + server.tool( + 'browser_dom_set_input_value', + { + selector: z.string().min(1).describe('CSS selector for the input/select/textarea element'), + value: z.string().describe('Value to assign to the matched element'), + simulateKeystrokes: z.boolean().optional().describe('Whether to emit a per-character key/input chain'), + blur: z.boolean().optional().describe('Whether to blur the element after setting the value'), + tabId: optionalTabIdSchema, + frameIds: optionalFrameIdsSchema, + }, + async ({ selector, value, simulateKeystrokes, blur, tabId, frameIds }) => { + if (!runtime.browserDomBridge.getStatus().connected) + return buildBrowserDomUnavailableResponse(runtime) + + const results = await runtime.browserDomBridge.setInputValue({ + selector, + value, + simulateKeystrokes, + blur, + tabId, + frameIds, + }) + return { + content: [ + textContent(summarizeBrowserDomFrameResults(`set_input_value for "${selector}"`, results)), + ], + structuredContent: { + status: 'ok', + selector, + valueLength: value.length, + results, + bridge: runtime.browserDomBridge.getStatus(), + }, + } + }, + ) + + server.tool( + 'browser_dom_check_checkbox', + { + selector: z.string().min(1).describe('CSS selector for the checkbox or radio-like element'), + checked: z.boolean().optional().describe('Target checked state; omit to toggle'), + tabId: optionalTabIdSchema, + frameIds: optionalFrameIdsSchema, + }, + async ({ selector, checked, tabId, frameIds }) => { + if (!runtime.browserDomBridge.getStatus().connected) + return buildBrowserDomUnavailableResponse(runtime) + + const results = await runtime.browserDomBridge.checkCheckbox({ + selector, + checked, + tabId, + frameIds, + }) + return { + content: [ + textContent(summarizeBrowserDomFrameResults(`check_checkbox for "${selector}"`, results)), + ], + structuredContent: { + status: 'ok', + selector, + checked, + results, + bridge: runtime.browserDomBridge.getStatus(), + }, + } + }, + ) + + server.tool( + 'browser_dom_select_option', + { + selector: z.string().min(1).describe('CSS selector for the