feat(stage-tamagotchi,computer-use-mcp): implement browser-native DOM action routing (#1648)
--------- Co-authored-by-agent: Antigravity <antigravity@gemini.com> Co-authored-by-agent: Codex <267193182+codex@users.noreply.github.com>
This commit is contained in:
刘梓恒
@@ -133,6 +133,11 @@ export default defineConfig({
|
||||
alias: {
|
||||
'@proj-airi/server-sdk': resolve(join(import.meta.dirname, '..', '..', 'packages', 'server-sdk', 'src')),
|
||||
'@proj-airi/i18n': resolve(join(import.meta.dirname, '..', '..', 'packages', 'i18n', 'src')),
|
||||
// NOTICE: the @proj-airi/stage-ui alias resolves to a directory; rolldown
|
||||
// concatenates sub-paths without a file extension, so bare .ts files at the
|
||||
// stores/ root (e.g. mcp-tool-bridge.ts) are not found. Add explicit aliases
|
||||
// for each such file that the renderer imports from @proj-airi/stage-ui.
|
||||
'@proj-airi/stage-ui/stores/mcp-tool-bridge': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-ui', 'src', 'stores', 'mcp-tool-bridge.ts')),
|
||||
'@proj-airi/stage-ui': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-ui', 'src')),
|
||||
'@proj-airi/stage-pages': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-pages', 'src')),
|
||||
'@proj-airi/stage-shared': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-shared', 'src')),
|
||||
|
||||
@@ -207,7 +207,7 @@ app.whenReady().then(async () => {
|
||||
// provider depends on 'windows:desktop-overlay'.
|
||||
injeca.invoke({
|
||||
dependsOn: { desktopOverlay },
|
||||
callback: noop,
|
||||
callback: () => {},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { McpCallToolResult } from '@proj-airi/stage-ui/tools/mcp'
|
||||
|
||||
import type { ElectronMcpCallToolResult } from '../../shared/eventa'
|
||||
import type { OverlayState } from './desktop-overlay-polling'
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
@@ -162,7 +161,7 @@ describe('createOverlayPollController', () => {
|
||||
it('calls tool and delivers state on successful poll', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const mockResult: McpCallToolResult = {
|
||||
const mockResult: ElectronMcpCallToolResult = {
|
||||
structuredContent: {
|
||||
runState: {
|
||||
lastGroundingSnapshot: {
|
||||
@@ -176,7 +175,7 @@ describe('createOverlayPollController', () => {
|
||||
},
|
||||
}
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
const callTool = vi.fn<(name: string) => Promise<ElectronMcpCallToolResult>>()
|
||||
.mockResolvedValue(mockResult)
|
||||
|
||||
const received: OverlayState[] = []
|
||||
@@ -201,32 +200,10 @@ describe('createOverlayPollController', () => {
|
||||
controller.stop()
|
||||
})
|
||||
|
||||
it('clears the per-call timeout when the tool resolves before the timeout fires', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
.mockResolvedValue({ structuredContent: {} })
|
||||
|
||||
const controller = createOverlayPollController({
|
||||
callTool,
|
||||
onState: () => {},
|
||||
intervalMs: 100,
|
||||
callTimeoutMs: 500,
|
||||
})
|
||||
|
||||
controller.start()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
// Only the next poll should remain scheduled. The per-call timeout must be cleared.
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
|
||||
controller.stop()
|
||||
})
|
||||
|
||||
it('stops polling after stop() is called', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
const callTool = vi.fn<(name: string) => Promise<ElectronMcpCallToolResult>>()
|
||||
.mockResolvedValue({ structuredContent: {} })
|
||||
|
||||
const controller = createOverlayPollController({
|
||||
@@ -250,7 +227,7 @@ describe('createOverlayPollController', () => {
|
||||
it('continues polling after a single failure', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
const callTool = vi.fn<(name: string) => Promise<ElectronMcpCallToolResult>>()
|
||||
.mockRejectedValueOnce(new Error('MCP down'))
|
||||
.mockResolvedValue({
|
||||
structuredContent: {
|
||||
@@ -292,7 +269,7 @@ describe('createOverlayPollController', () => {
|
||||
it('is a no-op to call start() twice', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
const callTool = vi.fn<(name: string) => Promise<ElectronMcpCallToolResult>>()
|
||||
.mockResolvedValue({ structuredContent: {} })
|
||||
|
||||
const controller = createOverlayPollController({
|
||||
@@ -313,8 +290,9 @@ describe('createOverlayPollController', () => {
|
||||
it('recovers from a hanging callTool via per-call timeout', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
.mockImplementationOnce(() => new Promise<McpCallToolResult>(() => {}))
|
||||
// First call hangs forever (simulates startup race when RPC not ready)
|
||||
const callTool = vi.fn<(name: string) => Promise<ElectronMcpCallToolResult>>()
|
||||
.mockImplementationOnce(() => new Promise(() => {})) // never resolves
|
||||
.mockResolvedValue({
|
||||
structuredContent: {
|
||||
runState: {
|
||||
@@ -344,10 +322,11 @@ describe('createOverlayPollController', () => {
|
||||
expect(callTool).toHaveBeenCalledTimes(1)
|
||||
expect(received).toHaveLength(0)
|
||||
|
||||
// Advance past the timeout and several fallback windows. The controller
|
||||
// should allow a bounded recovery retry even though the original invoke
|
||||
// is still hung in the background.
|
||||
// Advance past the 500ms timeout → catch triggers, schedules fallback
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
expect(received).toHaveLength(0)
|
||||
|
||||
// Advance past the 200ms fallback interval → second poll fires and succeeds
|
||||
await vi.advanceTimersByTimeAsync(200)
|
||||
expect(callTool).toHaveBeenCalledTimes(2)
|
||||
expect(received).toHaveLength(1)
|
||||
@@ -355,87 +334,4 @@ describe('createOverlayPollController', () => {
|
||||
|
||||
controller.stop()
|
||||
})
|
||||
|
||||
it('caps outstanding timed-out polls to avoid unbounded buildup', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
.mockImplementation(() => new Promise<McpCallToolResult>(() => {}))
|
||||
|
||||
const controller = createOverlayPollController({
|
||||
callTool,
|
||||
onState: () => {},
|
||||
intervalMs: 100,
|
||||
fallbackIntervalMs: 200,
|
||||
callTimeoutMs: 500,
|
||||
})
|
||||
|
||||
controller.start()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(callTool).toHaveBeenCalledTimes(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
await vi.advanceTimersByTimeAsync(200)
|
||||
expect(callTool).toHaveBeenCalledTimes(2)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
await vi.advanceTimersByTimeAsync(1000)
|
||||
expect(callTool).toHaveBeenCalledTimes(2)
|
||||
|
||||
controller.stop()
|
||||
})
|
||||
|
||||
it('recovers again once a timed-out hung-call slot lease expires', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const callTool = vi.fn<(name: string) => Promise<McpCallToolResult>>()
|
||||
.mockImplementationOnce(() => new Promise<McpCallToolResult>(() => {}))
|
||||
.mockImplementationOnce(() => new Promise<McpCallToolResult>(() => {}))
|
||||
.mockResolvedValue({
|
||||
structuredContent: {
|
||||
runState: {
|
||||
lastGroundingSnapshot: {
|
||||
snapshotId: 'dg_after_lease',
|
||||
targetCandidates: [],
|
||||
staleFlags: { screenshot: false, ax: false, chromeSemantic: false },
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const received: OverlayState[] = []
|
||||
|
||||
const controller = createOverlayPollController({
|
||||
callTool,
|
||||
onState: (state) => {
|
||||
received.push(state)
|
||||
},
|
||||
intervalMs: 100,
|
||||
fallbackIntervalMs: 200,
|
||||
callTimeoutMs: 500,
|
||||
hungCallLeaseMs: 1000,
|
||||
})
|
||||
|
||||
controller.start()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(callTool).toHaveBeenCalledTimes(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
await vi.advanceTimersByTimeAsync(200)
|
||||
expect(callTool).toHaveBeenCalledTimes(2)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
await vi.advanceTimersByTimeAsync(200)
|
||||
expect(callTool).toHaveBeenCalledTimes(2)
|
||||
expect(received).toHaveLength(0)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(200)
|
||||
expect(callTool).toHaveBeenCalledTimes(3)
|
||||
expect(received).toHaveLength(1)
|
||||
expect(received[0].snapshotId).toBe('dg_after_lease')
|
||||
|
||||
controller.stop()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* without a DOM environment or Vue test-utils.
|
||||
*/
|
||||
|
||||
import type { McpCallToolResult } from '@proj-airi/stage-ui/tools/mcp'
|
||||
import type { ElectronMcpCallToolResult } from '../../shared/eventa'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types — minimal shapes matching RunState fields the overlay consumes
|
||||
@@ -90,7 +90,7 @@ export function extractOverlayState(runState: Record<string, unknown>): OverlayS
|
||||
* Extract runState from an MCP call result.
|
||||
* Returns undefined if the result is an error or has no structured content.
|
||||
*/
|
||||
export function extractRunStateFromResult(result: McpCallToolResult): Record<string, unknown> | undefined {
|
||||
export function extractRunStateFromResult(result: ElectronMcpCallToolResult): Record<string, unknown> | undefined {
|
||||
if (result.isError)
|
||||
return undefined
|
||||
|
||||
@@ -121,7 +121,7 @@ export interface OverlayPollController {
|
||||
|
||||
export interface OverlayPollConfig {
|
||||
/** Function to call MCP tool. */
|
||||
callTool: (name: string) => Promise<McpCallToolResult>
|
||||
callTool: (name: string) => Promise<ElectronMcpCallToolResult>
|
||||
/** Callback with extracted state on each successful poll. */
|
||||
onState: (state: OverlayState) => void
|
||||
/** Normal poll interval in ms. Default: 250. */
|
||||
@@ -130,15 +130,11 @@ export interface OverlayPollConfig {
|
||||
fallbackIntervalMs?: number
|
||||
/** Per-call timeout in ms. Default: 5000. Prevents poll loop hang on startup race. */
|
||||
callTimeoutMs?: number
|
||||
/** How long a timed-out background call occupies a recovery slot before we probe again. */
|
||||
hungCallLeaseMs?: number
|
||||
}
|
||||
|
||||
const DEFAULT_INTERVAL = 250
|
||||
const DEFAULT_FALLBACK_INTERVAL = 500
|
||||
const DEFAULT_CALL_TIMEOUT = 5000
|
||||
const DEFAULT_HUNG_CALL_LEASE = 5000
|
||||
const MAX_BACKGROUND_HUNG_CALLS = 2
|
||||
|
||||
/**
|
||||
* MCP server name for computer-use-mcp. Matches the key in mcp.json.
|
||||
@@ -152,75 +148,21 @@ export const MCP_TOOL_NAME = 'computer_use::desktop_get_state'
|
||||
export function createOverlayPollController(config: OverlayPollConfig): OverlayPollController {
|
||||
const normalInterval = config.intervalMs ?? DEFAULT_INTERVAL
|
||||
const fallbackInterval = config.fallbackIntervalMs ?? DEFAULT_FALLBACK_INTERVAL
|
||||
const hungCallLeaseMs = config.hungCallLeaseMs ?? DEFAULT_HUNG_CALL_LEASE
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
let running = false
|
||||
let inFlightCall: Promise<McpCallToolResult> | null = null
|
||||
let backgroundHungSlots: Array<{ expiresAt: number }> = []
|
||||
|
||||
function scheduleNext(nextInterval: number) {
|
||||
if (running) {
|
||||
timer = setTimeout(poll, nextInterval)
|
||||
}
|
||||
}
|
||||
|
||||
function pruneHungCallSlots(now: number) {
|
||||
backgroundHungSlots = backgroundHungSlots.filter(slot => slot.expiresAt > now)
|
||||
}
|
||||
|
||||
async function poll() {
|
||||
pruneHungCallSlots(Date.now())
|
||||
|
||||
if (inFlightCall || backgroundHungSlots.length >= MAX_BACKGROUND_HUNG_CALLS) {
|
||||
scheduleNext(fallbackInterval)
|
||||
return
|
||||
}
|
||||
|
||||
let nextInterval = normalInterval
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
try {
|
||||
// NOTICE: Wrap callTool with a timeout to prevent the poll loop from
|
||||
// hanging forever if the eventa invoke never resolves (e.g. during
|
||||
// startup when the main-process RPC handlers may not be ready yet).
|
||||
// NOTICE: The bridge does not expose abort semantics, so a timed-out
|
||||
// call may still be pending in the background. We therefore track
|
||||
// timed-out calls as expiring lease slots: the cap bounds how many
|
||||
// unrecoverable invokes we tolerate at once, while lease expiry still
|
||||
// lets the overlay probe again after a cooling-off window.
|
||||
let timedOutSlot: { expiresAt: number } | null = null
|
||||
const currentCall = config.callTool(MCP_TOOL_NAME)
|
||||
inFlightCall = currentCall
|
||||
currentCall.then(() => {
|
||||
if (timedOutSlot) {
|
||||
backgroundHungSlots = backgroundHungSlots.filter(slot => slot !== timedOutSlot)
|
||||
}
|
||||
else if (inFlightCall === currentCall) {
|
||||
inFlightCall = null
|
||||
}
|
||||
}, () => {
|
||||
if (timedOutSlot) {
|
||||
backgroundHungSlots = backgroundHungSlots.filter(slot => slot !== timedOutSlot)
|
||||
}
|
||||
else if (inFlightCall === currentCall) {
|
||||
inFlightCall = null
|
||||
}
|
||||
})
|
||||
|
||||
const result = await Promise.race([
|
||||
currentCall,
|
||||
config.callTool(MCP_TOOL_NAME),
|
||||
new Promise<never>((_, reject) =>
|
||||
timeoutId = setTimeout(() => {
|
||||
timedOutSlot = {
|
||||
expiresAt: Date.now() + hungCallLeaseMs,
|
||||
}
|
||||
backgroundHungSlots = [...backgroundHungSlots, timedOutSlot]
|
||||
if (inFlightCall === currentCall) {
|
||||
inFlightCall = null
|
||||
}
|
||||
reject(new Error('callTool timeout'))
|
||||
}, config.callTimeoutMs ?? DEFAULT_CALL_TIMEOUT),
|
||||
setTimeout(() => reject(new Error('callTool timeout')), config.callTimeoutMs ?? DEFAULT_CALL_TIMEOUT),
|
||||
),
|
||||
])
|
||||
const runState = extractRunStateFromResult(result)
|
||||
@@ -236,13 +178,10 @@ export function createOverlayPollController(config: OverlayPollConfig): OverlayP
|
||||
// MCP server not running, bridge disconnected, or timeout — graceful degradation
|
||||
nextInterval = fallbackInterval
|
||||
}
|
||||
finally {
|
||||
if (timeoutId !== undefined) {
|
||||
clearTimeout(timeoutId)
|
||||
}
|
||||
}
|
||||
|
||||
scheduleNext(nextInterval)
|
||||
if (running) {
|
||||
timer = setTimeout(poll, nextInterval)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -25,11 +25,15 @@ import { pointInOverlay, rectIntersectsOverlay, screenRectToLocal, screenToLocal
|
||||
import { createEmptyOverlayState, createOverlayPollController } from './desktop-overlay-polling'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overlay window bounds
|
||||
// Overlay window bounds — read once on mount from main process
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const getWindowBounds = useElectronEventaInvoke(electron.window.getBounds)
|
||||
const callMcpTool = useElectronEventaInvoke(electronMcpCallTool)
|
||||
// Use Eventa invoke for MCP tool calls — McpToolBridge requires a
|
||||
// setMcpToolBridge() caller that does not exist in the overlay renderer.
|
||||
// electronMcpCallTool is already wired in setupDesktopOverlayElectronInvokes
|
||||
// via createMcpServersService, so it works without any extra bootstrap.
|
||||
const mcpCallTool = useElectronEventaInvoke(electronMcpCallTool)
|
||||
const overlayBounds = ref<Rect | null>(null)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -66,8 +70,12 @@ const matchedCandidate = computed(() => {
|
||||
return visibleCandidates.value.find(c => c.id === pointerIntent.value!.candidateId) ?? null
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Polling controller
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const controller = createOverlayPollController({
|
||||
callTool: name => callMcpTool({ name }),
|
||||
callTool: async name => mcpCallTool({ name }),
|
||||
onState: (newState) => {
|
||||
state.value = newState
|
||||
},
|
||||
@@ -123,11 +131,14 @@ const targetBoxStyle = computed(() => {
|
||||
// Lifecycle
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function syncOverlayBounds() {
|
||||
onMounted(async () => {
|
||||
// Read overlay window bounds from main process (one-time)
|
||||
try {
|
||||
overlayBounds.value = await getWindowBounds()
|
||||
const bounds = await getWindowBounds()
|
||||
overlayBounds.value = bounds
|
||||
}
|
||||
catch {
|
||||
// Fallback: assume bounds start at (0,0) with window inner size
|
||||
overlayBounds.value = {
|
||||
x: 0,
|
||||
y: 0,
|
||||
@@ -135,20 +146,11 @@ async function syncOverlayBounds() {
|
||||
height: window.innerHeight,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function handleResize() {
|
||||
void syncOverlayBounds()
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
await syncOverlayBounds()
|
||||
window.addEventListener('resize', handleResize)
|
||||
controller.start()
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
window.removeEventListener('resize', handleResize)
|
||||
controller.stop()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -39,6 +39,7 @@
|
||||
"./stores/settings/analytics": "./src/stores/settings/analytics.ts",
|
||||
"./stores/settings": "./src/stores/settings/index.ts",
|
||||
"./stores/modules/vision": "./src/stores/modules/vision/index.ts",
|
||||
"./stores/mcp-tool-bridge": "./src/stores/mcp-tool-bridge.ts",
|
||||
"./stores/*": "./src/stores/*.ts",
|
||||
"./stores": "./src/stores/index.ts",
|
||||
"./workers/vad": "./src/workers/vad/index.ts",
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Minimal bridge interface for calling MCP tools from the desktop overlay
|
||||
* renderer without a direct dependency on the MCP server runtime.
|
||||
*
|
||||
* The bridge is set by the Electron main/preload layer (or by a test stub)
|
||||
* and retrieved by overlay pages that need to invoke computer-use MCP tools.
|
||||
*/
|
||||
|
||||
export interface McpToolDescriptor {
|
||||
serverName: string
|
||||
name: string
|
||||
toolName: string
|
||||
description?: string
|
||||
inputSchema: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface McpCallToolPayload {
|
||||
name: string
|
||||
arguments?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface McpCallToolResult {
|
||||
content?: Array<Record<string, unknown>>
|
||||
structuredContent?: Record<string, unknown>
|
||||
toolResult?: unknown
|
||||
isError?: boolean
|
||||
}
|
||||
|
||||
interface McpToolBridge {
|
||||
listTools: () => Promise<McpToolDescriptor[]>
|
||||
callTool: (payload: McpCallToolPayload) => Promise<McpCallToolResult>
|
||||
}
|
||||
|
||||
let bridge: McpToolBridge | undefined
|
||||
|
||||
export function setMcpToolBridge(nextBridge: McpToolBridge) {
|
||||
bridge = nextBridge
|
||||
}
|
||||
|
||||
export function clearMcpToolBridge() {
|
||||
bridge = undefined
|
||||
}
|
||||
|
||||
export function getMcpToolBridge(): McpToolBridge {
|
||||
if (!bridge) {
|
||||
throw new Error('MCP tool bridge is not available in this runtime.')
|
||||
}
|
||||
|
||||
return bridge
|
||||
}
|
||||
@@ -1,21 +1,21 @@
|
||||
# AIRI Desktop Grounding — Chrome Extension
|
||||
|
||||
Read-only Chrome DOM observation bridge for the AIRI Desktop Grounding layer.
|
||||
Chrome DOM observation and interaction bridge for the AIRI Desktop Grounding layer.
|
||||
|
||||
## What it does
|
||||
|
||||
- Collects interactive elements (buttons, links, inputs, etc.) from all frames in the active Chrome tab
|
||||
- Reports element positions, ARIA roles, text, and rect coordinates
|
||||
- Feeds this data into the desktop grounding snap resolver for coordinate mapping
|
||||
- Performs targeted DOM interactions (set input values, check checkboxes, trigger events) when routed by the action executor
|
||||
|
||||
## What it does NOT do
|
||||
|
||||
- ❌ No DOM mutations (no clicking, typing, scrolling on DOM elements)
|
||||
- ❌ No `eval` / `new Function` / `chrome.scripting.executeScript`
|
||||
- ❌ No external network requests (no Python bridge, no offscreen documents)
|
||||
- ❌ No popup UI
|
||||
|
||||
All user interactions are performed via real macOS OS-level input events (CGEvent) through the desktop grounding executor.
|
||||
Physical click/type/scroll actions are performed via real macOS OS-level input events (CGEvent) through the desktop grounding executor. DOM mutations are limited to form-field writes and synthetic event dispatch via the bridge.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -27,6 +27,8 @@ msg_bridge.js (ISOLATED world)
|
||||
content.js (MAIN world, window.__AIRI_DG__)
|
||||
```
|
||||
|
||||
The background service worker also maintains a native WebSocket connection to `BrowserDomExtensionBridge` (default port 8765) to relay commands from the AIRI host process.
|
||||
|
||||
## Installation (development)
|
||||
|
||||
1. Open `chrome://extensions/`
|
||||
@@ -35,23 +37,6 @@ content.js (MAIN world, window.__AIRI_DG__)
|
||||
4. Select this `chrome-extension/` directory
|
||||
5. The extension will auto-inject into all pages
|
||||
|
||||
## Bridge endpoint override
|
||||
|
||||
By default the background worker connects to `ws://127.0.0.1:8765`.
|
||||
|
||||
If `computer-use-mcp` is running with a non-default
|
||||
`COMPUTER_USE_BROWSER_DOM_BRIDGE_HOST` or `COMPUTER_USE_BROWSER_DOM_BRIDGE_PORT`,
|
||||
override the extension endpoint through `chrome.storage.local`:
|
||||
|
||||
```js
|
||||
await chrome.storage.local.set({
|
||||
browserDomBridgeHost: '127.0.0.1',
|
||||
browserDomBridgePort: 8876,
|
||||
})
|
||||
```
|
||||
|
||||
The service worker watches these keys and reconnects automatically.
|
||||
|
||||
## Supported commands
|
||||
|
||||
| Command | Description |
|
||||
@@ -63,7 +48,15 @@ The service worker watches these keys and reconnects automatically.
|
||||
| `findElements` | Find multiple elements by CSS selector |
|
||||
| `getClickTarget` | Get element center point for click targeting |
|
||||
| `getElementAttributes` | Get all attributes of an element |
|
||||
| `setInputValue` | Set value of a text input or textarea |
|
||||
| `checkCheckbox` | Check or uncheck a native checkbox/radio |
|
||||
| `selectOption` | Select an option in a `<select>` element |
|
||||
| `readInputValue` | Read the current value of an input/textarea/select |
|
||||
| `getComputedStyles` | Get computed CSS styles for an element |
|
||||
| `triggerEvent` | Dispatch a DOM event on an element |
|
||||
| `waitForElement` | Wait for an element to appear in the DOM |
|
||||
| `clickAt` | Dispatch a click event at viewport coordinates |
|
||||
|
||||
## Provenance
|
||||
|
||||
Adapted from `/Users/liuziheng/computer_use/chrome-extension/` with DOM-action methods stripped.
|
||||
Adapted from the upstream computer-use chrome-extension.
|
||||
|
||||
@@ -12,165 +12,12 @@
|
||||
* All DOM-mutating actions (click, type, hover, scroll) have been removed
|
||||
* because the desktop lane uses real macOS OS-level input events.
|
||||
*
|
||||
* Adapted from /Users/liuziheng/computer_use/chrome-extension/background.js.
|
||||
* Adapted from the upstream computer-use chrome-extension.
|
||||
* Stripped: offscreen management, Python bridge, all DOM-action commands
|
||||
* (clickAt, typeAt, hoverAt, scrollAt, simulateDragDrop, readStorage,
|
||||
* setStorage, readCanvasData, injectCSS, executeScript, etc.)
|
||||
*/
|
||||
|
||||
// ---- Bridge connection ----
|
||||
|
||||
const DEFAULT_BRIDGE_HOST = '127.0.0.1'
|
||||
const DEFAULT_BRIDGE_PORT = 8765
|
||||
const BRIDGE_RECONNECT_DELAY_MS = 1000
|
||||
const BRIDGE_HOST_STORAGE_KEY = 'browserDomBridgeHost'
|
||||
const BRIDGE_PORT_STORAGE_KEY = 'browserDomBridgePort'
|
||||
|
||||
let bridgeSocket = null
|
||||
let reconnectTimer = null
|
||||
let connecting = false
|
||||
let bridgeHost = DEFAULT_BRIDGE_HOST
|
||||
let bridgePort = DEFAULT_BRIDGE_PORT
|
||||
|
||||
function clearReconnectTimer() {
|
||||
if (reconnectTimer !== null) {
|
||||
clearTimeout(reconnectTimer)
|
||||
reconnectTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleReconnect(delayMs = BRIDGE_RECONNECT_DELAY_MS) {
|
||||
if (reconnectTimer !== null)
|
||||
return
|
||||
|
||||
reconnectTimer = setTimeout(() => {
|
||||
reconnectTimer = null
|
||||
connectBridge().catch(() => {})
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
function sendBridgeMessage(payload) {
|
||||
if (!bridgeSocket || bridgeSocket.readyState !== WebSocket.OPEN)
|
||||
return false
|
||||
|
||||
bridgeSocket.send(JSON.stringify(payload))
|
||||
return true
|
||||
}
|
||||
|
||||
function normalizeBridgeHost(value) {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : DEFAULT_BRIDGE_HOST
|
||||
}
|
||||
|
||||
function normalizeBridgePort(value) {
|
||||
if (typeof value === 'number' && Number.isInteger(value) && value > 0)
|
||||
return value
|
||||
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
const parsed = Number.parseInt(value.trim(), 10)
|
||||
if (Number.isInteger(parsed) && parsed > 0)
|
||||
return parsed
|
||||
}
|
||||
|
||||
return DEFAULT_BRIDGE_PORT
|
||||
}
|
||||
|
||||
async function loadBridgeConfig() {
|
||||
try {
|
||||
const stored = await chrome.storage.local.get([
|
||||
BRIDGE_HOST_STORAGE_KEY,
|
||||
BRIDGE_PORT_STORAGE_KEY,
|
||||
])
|
||||
bridgeHost = normalizeBridgeHost(stored[BRIDGE_HOST_STORAGE_KEY])
|
||||
bridgePort = normalizeBridgePort(stored[BRIDGE_PORT_STORAGE_KEY])
|
||||
}
|
||||
catch {
|
||||
bridgeHost = DEFAULT_BRIDGE_HOST
|
||||
bridgePort = DEFAULT_BRIDGE_PORT
|
||||
}
|
||||
}
|
||||
|
||||
async function saveBridgeConfig(host, port) {
|
||||
await chrome.storage.local.set({
|
||||
[BRIDGE_HOST_STORAGE_KEY]: normalizeBridgeHost(host),
|
||||
[BRIDGE_PORT_STORAGE_KEY]: normalizeBridgePort(port),
|
||||
})
|
||||
await loadBridgeConfig()
|
||||
}
|
||||
|
||||
async function handleBridgeMessage(raw) {
|
||||
let data
|
||||
try {
|
||||
data = JSON.parse(String(raw))
|
||||
}
|
||||
catch {
|
||||
return
|
||||
}
|
||||
|
||||
const response = await handleCommand(data)
|
||||
sendBridgeMessage(response)
|
||||
}
|
||||
|
||||
async function connectBridge() {
|
||||
if (connecting)
|
||||
return
|
||||
if (bridgeSocket && (bridgeSocket.readyState === WebSocket.OPEN || bridgeSocket.readyState === WebSocket.CONNECTING))
|
||||
return
|
||||
|
||||
connecting = true
|
||||
try {
|
||||
await loadBridgeConfig()
|
||||
const socket = new WebSocket(`ws://${bridgeHost}:${bridgePort}`)
|
||||
bridgeSocket = socket
|
||||
|
||||
socket.addEventListener('open', () => {
|
||||
connecting = false
|
||||
clearReconnectTimer()
|
||||
sendBridgeMessage({
|
||||
type: 'hello',
|
||||
source: 'airi-desktop-grounding-extension',
|
||||
version: chrome.runtime.getManifest().version,
|
||||
})
|
||||
})
|
||||
|
||||
socket.addEventListener('message', (event) => {
|
||||
void handleBridgeMessage(event.data)
|
||||
})
|
||||
|
||||
socket.addEventListener('close', () => {
|
||||
if (bridgeSocket === socket) {
|
||||
bridgeSocket = null
|
||||
}
|
||||
connecting = false
|
||||
scheduleReconnect()
|
||||
})
|
||||
|
||||
socket.addEventListener('error', () => {
|
||||
connecting = false
|
||||
try {
|
||||
socket.close()
|
||||
}
|
||||
catch {}
|
||||
})
|
||||
}
|
||||
catch {
|
||||
connecting = false
|
||||
scheduleReconnect()
|
||||
}
|
||||
}
|
||||
|
||||
function reconnectBridgeNow() {
|
||||
clearReconnectTimer()
|
||||
if (bridgeSocket) {
|
||||
try {
|
||||
bridgeSocket.close()
|
||||
}
|
||||
catch {}
|
||||
bridgeSocket = null
|
||||
}
|
||||
connecting = false
|
||||
void connectBridge()
|
||||
}
|
||||
|
||||
// ---- Tab / Frame utilities ----
|
||||
|
||||
async function getActiveTab() {
|
||||
@@ -240,7 +87,7 @@ async function runCUAction(tabId, frameIds, method, args) {
|
||||
/**
|
||||
* Handle a command from the AIRI BrowserDomExtensionBridge.
|
||||
*
|
||||
* Only read-only observation commands are supported:
|
||||
* Supported actions:
|
||||
* - getActiveTab: get the active tab info
|
||||
* - getAllFrames: list all frames in the active tab
|
||||
* - readAllFramesDOM: collect interactive elements from all frames
|
||||
@@ -248,6 +95,14 @@ async function runCUAction(tabId, frameIds, method, args) {
|
||||
* - findElements: find multiple elements by CSS selector
|
||||
* - getClickTarget: get center point of an element for click targeting
|
||||
* - getElementAttributes: get all attributes of an element
|
||||
* - setInputValue: set value of a text input or textarea
|
||||
* - checkCheckbox: check or uncheck a native checkbox/radio
|
||||
* - selectOption: select an option in a <select> element
|
||||
* - readInputValue: read the current value of an input/textarea/select
|
||||
* - getComputedStyles: get computed CSS styles for an element
|
||||
* - triggerEvent: dispatch a DOM event on an element
|
||||
* - waitForElement: wait for an element to appear in the DOM
|
||||
* - clickAt: dispatch a click event at viewport coordinates
|
||||
*/
|
||||
async function handleCommand(cmd) {
|
||||
const { action, id } = cmd
|
||||
@@ -289,7 +144,67 @@ async function handleCommand(cmd) {
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'getElementAttributes', [cmd.selector || ''])
|
||||
break
|
||||
|
||||
case 'setInputValue':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'setInputValue', [
|
||||
cmd.selector || '',
|
||||
cmd.value || '',
|
||||
{ blur: cmd.opts?.blur !== false, simulateKeystrokes: !!cmd.opts?.simulateKeystrokes },
|
||||
])
|
||||
break
|
||||
|
||||
case 'checkCheckbox':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'checkCheckbox', [
|
||||
cmd.selector || '',
|
||||
cmd.checked,
|
||||
])
|
||||
break
|
||||
|
||||
case 'selectOption':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'selectOption', [
|
||||
cmd.selector || '',
|
||||
cmd.value || '',
|
||||
])
|
||||
break
|
||||
|
||||
case 'readInputValue':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'readInputValue', [
|
||||
cmd.selector || '',
|
||||
])
|
||||
break
|
||||
|
||||
case 'getComputedStyles':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'getComputedStyles', [
|
||||
cmd.selector || '',
|
||||
cmd.properties || [],
|
||||
])
|
||||
break
|
||||
|
||||
case 'triggerEvent':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'triggerEvent', [
|
||||
cmd.selector || '',
|
||||
cmd.eventName || '',
|
||||
cmd.opts || {},
|
||||
])
|
||||
break
|
||||
|
||||
case 'waitForElement':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'waitForElement', [
|
||||
cmd.selector || '',
|
||||
cmd.timeoutMs || 5000,
|
||||
])
|
||||
break
|
||||
|
||||
case 'clickAt':
|
||||
result = await runCUAction(tabId, cmd.frameIds || null, 'clickAt', [
|
||||
cmd.x ?? 0,
|
||||
cmd.y ?? 0,
|
||||
])
|
||||
break
|
||||
|
||||
default:
|
||||
// NOTICE: unknown actions must return ok:false so BrowserDomExtensionBridge
|
||||
// rejects the pending promise; returning ok:true would make callers like
|
||||
// setInputValue/checkCheckbox see a resolved promise and skip fallback paths.
|
||||
return { id, ok: false, error: `unknown action: ${action}` }
|
||||
}
|
||||
|
||||
@@ -305,24 +220,6 @@ async function handleCommand(cmd) {
|
||||
// or through the existing WebSocket bridge mechanism
|
||||
|
||||
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
|
||||
void connectBridge()
|
||||
|
||||
if (msg.type === 'AIRI_DG_SET_BRIDGE_ENDPOINT') {
|
||||
saveBridgeConfig(msg.host, msg.port)
|
||||
.then(() => {
|
||||
reconnectBridgeNow()
|
||||
sendResponse({
|
||||
ok: true,
|
||||
host: bridgeHost,
|
||||
port: bridgePort,
|
||||
})
|
||||
})
|
||||
.catch((e) => {
|
||||
sendResponse({ ok: false, error: e?.message || String(e) })
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
if (msg.type === 'AIRI_DG_COMMAND') {
|
||||
handleCommand(msg.data)
|
||||
.then(resp => sendResponse(resp))
|
||||
@@ -330,39 +227,64 @@ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
|
||||
return true // Keep sendResponse async
|
||||
}
|
||||
|
||||
// Support the existing ws-incoming format from BrowserDomExtensionBridge
|
||||
if (msg.type === 'ws-incoming') {
|
||||
handleCommand(msg.data)
|
||||
.then((resp) => {
|
||||
// Send response back via the same channel
|
||||
chrome.runtime.sendMessage({ type: 'ws-send', data: resp })
|
||||
})
|
||||
.catch((e) => {
|
||||
chrome.runtime.sendMessage({ type: 'ws-send', data: { id: msg.data?.id, ok: false, error: String(e) } })
|
||||
})
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
})
|
||||
|
||||
chrome.storage.onChanged.addListener((changes, areaName) => {
|
||||
if (areaName !== 'local')
|
||||
// ---- WebSocket Relay ----
|
||||
// Injects the WebSocket connection directly in the background worker,
|
||||
// replacing the deleted offscreen document.
|
||||
// TODO: Add shared-secret auth handshake to prevent rogue localhost processes
|
||||
// from hijacking the bridge. The bridge server should generate a token and
|
||||
// inject it into chrome.storage.local so the extension can present it on hello.
|
||||
const WS_URL = 'ws://localhost:8765'
|
||||
const BRIDGE_VERSION = 'cu-bridge-2026-02-06-no-eval'
|
||||
let ws = null
|
||||
let reconnectDelay = 1000
|
||||
const MAX_DELAY = 30000
|
||||
|
||||
function connectWS() {
|
||||
if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING))
|
||||
return
|
||||
|
||||
if (changes[BRIDGE_HOST_STORAGE_KEY] || changes[BRIDGE_PORT_STORAGE_KEY]) {
|
||||
void loadBridgeConfig().finally(() => {
|
||||
reconnectBridgeNow()
|
||||
})
|
||||
ws = new WebSocket(WS_URL)
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('[background] WebSocket connected')
|
||||
reconnectDelay = 1000
|
||||
ws.send(JSON.stringify({ type: 'hello', source: 'chrome-extension', version: BRIDGE_VERSION }))
|
||||
}
|
||||
})
|
||||
|
||||
chrome.runtime.onStartup.addListener(() => {
|
||||
void connectBridge()
|
||||
})
|
||||
ws.onmessage = (evt) => {
|
||||
try {
|
||||
const data = JSON.parse(evt.data)
|
||||
handleCommand(data)
|
||||
.then((resp) => {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify(resp))
|
||||
}
|
||||
})
|
||||
.catch((e) => {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ id: data?.id, ok: false, error: String(e) }))
|
||||
}
|
||||
})
|
||||
}
|
||||
catch (e) {
|
||||
console.error('[background] parse error:', e)
|
||||
}
|
||||
}
|
||||
|
||||
chrome.runtime.onInstalled.addListener(() => {
|
||||
void connectBridge()
|
||||
})
|
||||
ws.onclose = () => {
|
||||
console.log(`[background] WebSocket closed, reconnect in ${reconnectDelay}ms`)
|
||||
ws = null
|
||||
setTimeout(connectWS, reconnectDelay)
|
||||
reconnectDelay = Math.min(reconnectDelay * 2, MAX_DELAY)
|
||||
}
|
||||
|
||||
void connectBridge()
|
||||
ws.onerror = (e) => {
|
||||
console.error('[background] WebSocket error:', e)
|
||||
ws?.close()
|
||||
}
|
||||
}
|
||||
|
||||
connectWS()
|
||||
|
||||
@@ -4,15 +4,17 @@
|
||||
* Injected into every frame (including cross-origin iframes) in the MAIN world.
|
||||
* Namespace: window.__AIRI_DG__
|
||||
*
|
||||
* IMPORTANT: This script is READ-ONLY. It does NOT perform any DOM mutations,
|
||||
* clicks, typing, or navigation. All execution is done via real macOS OS-level
|
||||
* input events through the desktop grounding executor.
|
||||
* IMPORTANT: Direct DOM mutations here are limited to bridge-triggered write
|
||||
* actions (setInputValue, checkCheckbox, selectOption) that are only reachable
|
||||
* via a WebSocket command from the AIRI computer-use-mcp service. Physical
|
||||
* pointer/keyboard actions still go through real macOS OS-level input.
|
||||
*
|
||||
* Adapted from /Users/liuziheng/computer_use/chrome-extension/content.js.
|
||||
* Adapted from the upstream computer-use chrome-extension.
|
||||
* Stripped: clickAt, typeAt, hoverAt, scrollAt, simulateDragDrop, readStorage,
|
||||
* setStorage, readCanvasData, injectCSS, and all other DOM-mutating methods.
|
||||
* setStorage, readCanvasData, injectCSS, and all other untracked DOM mutations.
|
||||
* Kept: collectFrameDOM, _describeElement, _collectInteractiveElements,
|
||||
* findElement, findElements, getClickTarget.
|
||||
* Added: setInputValue, checkCheckbox, selectOption.
|
||||
*/
|
||||
(function () {
|
||||
'use strict'
|
||||
@@ -37,6 +39,7 @@
|
||||
name: el.name || '',
|
||||
type: el.type || '',
|
||||
className: typeof el.className === 'string' ? el.className.slice(0, 120) : '',
|
||||
// eslint-disable-next-line unicorn/prefer-dom-node-text-content -- intentional: innerText returns visible text only
|
||||
text: (el.innerText || el.textContent || '').slice(0, 120).trim(),
|
||||
value: el.value !== undefined ? String(el.value).slice(0, 60) : '',
|
||||
href: el.href || '',
|
||||
@@ -67,35 +70,6 @@
|
||||
return els
|
||||
}
|
||||
|
||||
/**
|
||||
* Get this frame's embedding rect relative to its parent viewport.
|
||||
*
|
||||
* NOTICE: Cross-origin frames may not expose `window.frameElement`.
|
||||
* In that case we return null and let the adapter skip those frame-local
|
||||
* coordinates rather than projecting them incorrectly onto the desktop.
|
||||
*/
|
||||
function _getFrameRect() {
|
||||
try {
|
||||
if (window.top === window)
|
||||
return null
|
||||
|
||||
const frameEl = window.frameElement
|
||||
if (!(frameEl instanceof Element))
|
||||
return null
|
||||
|
||||
const r = frameEl.getBoundingClientRect()
|
||||
return {
|
||||
x: Math.round(r.left),
|
||||
y: Math.round(r.top),
|
||||
w: Math.round(r.width),
|
||||
h: Math.round(r.height),
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Core API (read-only) ----
|
||||
|
||||
const __AIRI_DG__ = {
|
||||
@@ -112,8 +86,8 @@
|
||||
return {
|
||||
url: location.href,
|
||||
title: document.title || '',
|
||||
// eslint-disable-next-line unicorn/prefer-dom-node-text-content -- intentional: innerText returns visible text only
|
||||
bodyText: includeText ? (document.body ? document.body.innerText || '' : '').slice(0, 3000) : '',
|
||||
frameRect: _getFrameRect() || undefined,
|
||||
interactiveElements: _collectInteractiveElements(maxElements),
|
||||
}
|
||||
},
|
||||
@@ -156,6 +130,10 @@
|
||||
/**
|
||||
* Get the center point of an element for click targeting.
|
||||
* Returns the element description with center coordinates.
|
||||
*
|
||||
* Coordinates are exposed both at the top level (x, y) and under
|
||||
* `center` for backward compatibility. The extension bridge reads
|
||||
* top-level x/y via unwrapResultPayload.
|
||||
*/
|
||||
getClickTarget(selector) {
|
||||
try {
|
||||
@@ -163,13 +141,16 @@
|
||||
if (!el)
|
||||
return { success: false, error: 'not found' }
|
||||
const r = el.getBoundingClientRect()
|
||||
const x = Math.round(r.left + r.width / 2)
|
||||
const y = Math.round(r.top + r.height / 2)
|
||||
return {
|
||||
success: true,
|
||||
element: _describeElement(el),
|
||||
center: {
|
||||
x: Math.round(r.left + r.width / 2),
|
||||
y: Math.round(r.top + r.height / 2),
|
||||
},
|
||||
// Top-level x/y are read by extension-bridge.ts → clickSelector
|
||||
x,
|
||||
y,
|
||||
// Keep center for any callers that read it directly
|
||||
center: { x, y },
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
@@ -195,12 +176,227 @@
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Set the value of a text input or textarea via the DOM.
|
||||
* Dispatches input + change events so frameworks (React, Vue, etc.) detect
|
||||
* the change. Optionally blurs the element when done.
|
||||
*/
|
||||
setInputValue(selector, value, opts) {
|
||||
try {
|
||||
opts = opts || {}
|
||||
// TODO: opts.simulateKeystrokes is accepted but ignored — we always do
|
||||
// a single direct value assignment. Implement per-character KeyboardEvent
|
||||
// dispatch for autocomplete/masker/validation flows that depend on keydown/keyup.
|
||||
const el = document.querySelector(selector)
|
||||
if (!el)
|
||||
return { success: false, error: 'not found' }
|
||||
// NOTICE: must pick the setter matching the element's prototype —
|
||||
// calling HTMLInputElement.prototype.value.set on a <textarea> (or
|
||||
// vice-versa) throws "Illegal invocation" in Chromium.
|
||||
const proto = el instanceof HTMLTextAreaElement
|
||||
? window.HTMLTextAreaElement.prototype
|
||||
: window.HTMLInputElement.prototype
|
||||
const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')
|
||||
if (nativeInputValueSetter && nativeInputValueSetter.set) {
|
||||
nativeInputValueSetter.set.call(el, value)
|
||||
}
|
||||
else {
|
||||
el.value = value
|
||||
}
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }))
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }))
|
||||
if (opts.blur)
|
||||
el.blur()
|
||||
return { success: true }
|
||||
}
|
||||
catch (e) {
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Check or uncheck a native checkbox or radio input.
|
||||
* Sets checked programmatically and dispatches a change event so framework
|
||||
* bindings (React onChange, Vue @change) pick up the update.
|
||||
*
|
||||
* NOTICE: only works on real <input type="checkbox|radio"> elements.
|
||||
* Custom ARIA checkboxes (e.g. <div role="checkbox">) do not have a native
|
||||
* .checked property — writing to it just adds an expando attribute and
|
||||
* changes nothing visible. Return success:false in that case so the caller
|
||||
* falls back to an OS-level click.
|
||||
*
|
||||
* NOTICE: we do NOT dispatch a fake click event — the browser's true event
|
||||
* order is click→change, and a synthetic click after we've already set
|
||||
* el.checked can cause React controlled-component handlers to toggle back.
|
||||
*/
|
||||
checkCheckbox(selector, checked) {
|
||||
try {
|
||||
const el = document.querySelector(selector)
|
||||
if (!el)
|
||||
return { success: false, error: 'not found' }
|
||||
// Guard: only native checkbox/radio inputs have a meaningful .checked
|
||||
if (!(el instanceof HTMLInputElement) || (el.type !== 'checkbox' && el.type !== 'radio'))
|
||||
return { success: false, error: 'not a native checkbox or radio input' }
|
||||
const target = checked !== undefined ? !!checked : !el.checked
|
||||
if (el.checked !== target) {
|
||||
el.checked = target
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }))
|
||||
}
|
||||
return { success: true, checked: el.checked }
|
||||
}
|
||||
catch (e) {
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Select an option in a <select> element by value.
|
||||
* Dispatches change event so framework bindings update.
|
||||
*/
|
||||
selectOption(selector, value) {
|
||||
try {
|
||||
const el = document.querySelector(selector)
|
||||
if (!el)
|
||||
return { success: false, error: 'not found' }
|
||||
el.value = value
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }))
|
||||
return { success: true, selectedValue: el.value }
|
||||
}
|
||||
catch (e) {
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Read the current value of an input, textarea, or select element.
|
||||
*/
|
||||
readInputValue(selector) {
|
||||
try {
|
||||
const el = document.querySelector(selector)
|
||||
if (!el)
|
||||
return { success: false, error: 'not found' }
|
||||
return { success: true, value: el.value, tagName: el.tagName.toLowerCase() }
|
||||
}
|
||||
catch (e) {
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Get computed CSS styles for an element.
|
||||
* If properties is a non-empty array, only those properties are returned.
|
||||
* Otherwise all computed styles are returned.
|
||||
*/
|
||||
getComputedStyles(selector, properties) {
|
||||
try {
|
||||
const el = document.querySelector(selector)
|
||||
if (!el)
|
||||
return { success: false, error: 'not found' }
|
||||
const computed = window.getComputedStyle(el)
|
||||
const styles = {}
|
||||
if (Array.isArray(properties) && properties.length > 0) {
|
||||
for (const prop of properties) {
|
||||
styles[prop] = computed.getPropertyValue(prop)
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Return a small useful subset to avoid serializing 300+ properties
|
||||
const useful = ['display', 'visibility', 'opacity', 'position', 'width', 'height', 'color', 'background-color', 'font-size', 'overflow', 'pointer-events', 'z-index', 'cursor']
|
||||
for (const prop of useful) {
|
||||
styles[prop] = computed.getPropertyValue(prop)
|
||||
}
|
||||
}
|
||||
return { success: true, styles }
|
||||
}
|
||||
catch (e) {
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Dispatch a DOM event on the element matching the selector.
|
||||
* opts.type overrides the Event constructor (default: 'Event').
|
||||
*/
|
||||
triggerEvent(selector, eventName, opts) {
|
||||
try {
|
||||
opts = opts || {}
|
||||
const el = document.querySelector(selector)
|
||||
if (!el)
|
||||
return { success: false, error: 'not found' }
|
||||
const EventCtor = opts.type === 'MouseEvent'
|
||||
? MouseEvent
|
||||
: opts.type === 'KeyboardEvent'
|
||||
? KeyboardEvent
|
||||
: opts.type === 'FocusEvent'
|
||||
? FocusEvent
|
||||
: Event
|
||||
const eventOpts = { bubbles: true, cancelable: true, ...opts }
|
||||
delete eventOpts.type
|
||||
el.dispatchEvent(new EventCtor(eventName, eventOpts))
|
||||
return { success: true }
|
||||
}
|
||||
catch (e) {
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Wait for an element matching the selector to appear in the DOM.
|
||||
* Returns a promise. The message handler awaits it.
|
||||
*/
|
||||
waitForElement(selector, timeoutMs) {
|
||||
timeoutMs = timeoutMs || 5000
|
||||
const existing = document.querySelector(selector)
|
||||
if (existing)
|
||||
return { success: true, found: true }
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let timer
|
||||
const observer = new MutationObserver(() => {
|
||||
if (document.querySelector(selector)) {
|
||||
observer.disconnect()
|
||||
clearTimeout(timer)
|
||||
resolve({ success: true, found: true })
|
||||
}
|
||||
})
|
||||
observer.observe(document.documentElement, { childList: true, subtree: true })
|
||||
timer = setTimeout(() => {
|
||||
observer.disconnect()
|
||||
resolve({ success: false, error: 'timeout' })
|
||||
}, timeoutMs)
|
||||
})
|
||||
},
|
||||
|
||||
/**
|
||||
* Dispatch a click event at viewport coordinates (x, y).
|
||||
* Used by clickSelector as the final step after getClickTarget resolves
|
||||
* the element center.
|
||||
*/
|
||||
clickAt(x, y) {
|
||||
try {
|
||||
const el = document.elementFromPoint(x, y)
|
||||
if (!el)
|
||||
return { success: false, error: 'no element at point' }
|
||||
el.dispatchEvent(new MouseEvent('click', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: x,
|
||||
clientY: y,
|
||||
}))
|
||||
return { success: true, tagName: el.tagName.toLowerCase() }
|
||||
}
|
||||
catch (e) {
|
||||
return { success: false, error: e.message }
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
window.__AIRI_DG__ = __AIRI_DG__
|
||||
|
||||
// ---- Message handler: ISOLATED world bridge → MAIN world ----
|
||||
window.addEventListener('message', (evt) => {
|
||||
// NOTICE: handler is async-aware so waitForElement (returns Promise) works.
|
||||
window.addEventListener('message', async (evt) => {
|
||||
if (evt.source !== window)
|
||||
return
|
||||
const data = evt.data
|
||||
@@ -213,7 +409,13 @@
|
||||
|
||||
if (typeof fn === 'function') {
|
||||
try {
|
||||
result = { success: true, data: fn.apply(__AIRI_DG__, args || []) }
|
||||
const ret = fn.apply(__AIRI_DG__, args || [])
|
||||
// Support async methods (e.g. waitForElement)
|
||||
// NOTICE: return the method result directly — each method already
|
||||
// returns its own { success, data/error } shape. Wrapping it again
|
||||
// as { success: true, data: <result> } created a double-envelope
|
||||
// that made transport-level success hide DOM-level failures.
|
||||
result = ret && typeof ret.then === 'function' ? await ret : ret
|
||||
}
|
||||
catch (e) {
|
||||
result = { success: false, error: e.message || String(e) }
|
||||
|
||||
@@ -7,8 +7,7 @@
|
||||
"permissions": [
|
||||
"activeTab",
|
||||
"tabs",
|
||||
"webNavigation",
|
||||
"storage"
|
||||
"webNavigation"
|
||||
],
|
||||
"host_permissions": ["<all_urls>"],
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
* - window.__AIRI_DG__ lives in the MAIN world (needs real DOM access)
|
||||
* - The two worlds communicate via window.postMessage
|
||||
*
|
||||
* Adapted from /Users/liuziheng/computer_use/chrome-extension/msg_bridge.js.
|
||||
* Adapted from the upstream computer-use chrome-extension.
|
||||
* No functional changes — this is a pure relay.
|
||||
*/
|
||||
(function () {
|
||||
@@ -26,17 +26,13 @@
|
||||
// Pending requests: reqId → { sendResponse, timer }
|
||||
const pending = new Map()
|
||||
let seqId = 0
|
||||
const pageOrigin = window.location.origin
|
||||
const postMessageTargetOrigin = pageOrigin && pageOrigin !== 'null' ? pageOrigin : '*'
|
||||
|
||||
// Receive commands from background.js
|
||||
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
|
||||
if (msg.type !== 'CU_ACTION')
|
||||
return false
|
||||
|
||||
const reqId = typeof crypto?.randomUUID === 'function'
|
||||
? `__cu_req_${crypto.randomUUID()}`
|
||||
: `__cu_req_${++seqId}`
|
||||
const reqId = `__cu_req_${++seqId}`
|
||||
const { method, args } = msg
|
||||
|
||||
// Set timeout
|
||||
@@ -53,7 +49,7 @@
|
||||
reqId,
|
||||
method,
|
||||
args: args || [],
|
||||
}, postMessageTargetOrigin)
|
||||
}, '*')
|
||||
|
||||
return true // Keep sendResponse async
|
||||
})
|
||||
@@ -62,8 +58,6 @@
|
||||
window.addEventListener('message', (evt) => {
|
||||
if (evt.source !== window)
|
||||
return
|
||||
if (pageOrigin && pageOrigin !== 'null' && evt.origin !== pageOrigin)
|
||||
return
|
||||
const data = evt.data
|
||||
if (!data || data.type !== '__CU_REPLY__')
|
||||
return
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
import type { DesktopTargetCandidate } from './desktop-grounding-types'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decideBrowserAction, decideBrowserTypeAction } from './browser-action-router'
|
||||
|
||||
function makeCandidate(overrides: Partial<DesktopTargetCandidate> = {}): DesktopTargetCandidate {
|
||||
return {
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
appName: 'Google Chrome',
|
||||
role: 'button',
|
||||
label: 'Submit',
|
||||
bounds: { x: 100, y: 200, width: 80, height: 30 },
|
||||
confidence: 0.95,
|
||||
interactable: true,
|
||||
selector: '#submit-btn',
|
||||
frameId: 0,
|
||||
isPageContent: true,
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// decideBrowserAction (click routing)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('decideBrowserAction', () => {
|
||||
it('routes chrome_dom with selector + bridge available to browser_dom', () => {
|
||||
const decision = decideBrowserAction(makeCandidate(), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.selector).toBe('#submit-btn')
|
||||
expect(decision.frameId).toBe(0)
|
||||
expect(decision.bridgeMethod).toBe('clickSelector')
|
||||
})
|
||||
|
||||
it('falls back to os_input when source is ax', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({ source: 'ax' }), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('ax')
|
||||
})
|
||||
|
||||
it('falls back to os_input when source is vision', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({ source: 'vision' }), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
})
|
||||
|
||||
it('falls back to os_input when selector is missing', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({ selector: undefined }), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('no CSS selector')
|
||||
})
|
||||
|
||||
it('falls back to os_input when bridge is unavailable', () => {
|
||||
const decision = decideBrowserAction(makeCandidate(), false)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not connected')
|
||||
})
|
||||
|
||||
it('preserves non-zero frameId for sub-frame candidates', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({ frameId: 3 }), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.frameId).toBe(3)
|
||||
})
|
||||
|
||||
it('falls back to os_input when selector is empty string', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({ selector: '' }), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('no CSS selector')
|
||||
})
|
||||
|
||||
it('routes checkbox to checkCheckbox instead of clickSelector', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'checkbox',
|
||||
selector: '#agree-checkbox',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('checkCheckbox')
|
||||
expect(decision.selector).toBe('#agree-checkbox')
|
||||
})
|
||||
|
||||
it('routes checkbox by role to checkCheckbox', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({
|
||||
tag: 'div',
|
||||
role: 'checkbox',
|
||||
selector: 'div.custom-checkbox',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('checkCheckbox')
|
||||
})
|
||||
|
||||
it('routes regular button to clickSelector, not checkCheckbox', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({
|
||||
tag: 'button',
|
||||
role: 'button',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('clickSelector')
|
||||
})
|
||||
|
||||
it('routes radio input to clickSelector, not checkCheckbox', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'radio',
|
||||
selector: 'input[name="color"]',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('clickSelector')
|
||||
})
|
||||
|
||||
it('routes link element to clickSelector', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({
|
||||
tag: 'a',
|
||||
role: 'link',
|
||||
href: 'https://example.com',
|
||||
selector: 'a.nav-link',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('clickSelector')
|
||||
})
|
||||
|
||||
it('checkbox with bridge down falls back to os_input, not checkCheckbox', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'checkbox',
|
||||
selector: '#agree',
|
||||
}), false)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.bridgeMethod).toBeUndefined()
|
||||
})
|
||||
|
||||
it('returns reason string that includes the selector', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({ selector: '#my-btn' }), true)
|
||||
expect(decision.reason).toContain('#my-btn')
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// decideBrowserTypeAction (type routing)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('decideBrowserTypeAction', () => {
|
||||
it('routes text input to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'text',
|
||||
selector: 'input[name="email"]',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
expect(decision.selector).toBe('input[name="email"]')
|
||||
})
|
||||
|
||||
it('routes password input to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'password',
|
||||
selector: '#password',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('routes textarea to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'textarea',
|
||||
selector: '#message',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('routes input without explicit type (defaults to text) to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: undefined,
|
||||
selector: '#name',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('routes contenteditable via role=textbox to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'div',
|
||||
role: 'textbox',
|
||||
selector: 'div.editor',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('falls back to os_input for button elements', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'button',
|
||||
role: 'button',
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
|
||||
it('falls back to os_input for checkbox inputs', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'checkbox',
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
|
||||
it('falls back to os_input for file inputs', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'file',
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
|
||||
it('falls back to os_input when source is ax', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({ source: 'ax' }), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
})
|
||||
|
||||
it('falls back to os_input when bridge is unavailable', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'text',
|
||||
}), false)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not connected')
|
||||
})
|
||||
|
||||
it('falls back to os_input when selector is missing', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'text',
|
||||
selector: undefined,
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('no CSS selector')
|
||||
})
|
||||
|
||||
it('routes number input to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'number',
|
||||
selector: '#quantity',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('routes search input to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'search',
|
||||
selector: '#search',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('routes url input to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'url',
|
||||
selector: '#website',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('routes tel input to setInputValue', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'tel',
|
||||
selector: '#phone',
|
||||
}), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
})
|
||||
|
||||
it('falls back to os_input for radio inputs', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'radio',
|
||||
selector: 'input[name="option"]',
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
|
||||
it('falls back to os_input for hidden inputs', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'hidden',
|
||||
selector: '#csrf',
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
|
||||
it('falls back to os_input for color picker inputs', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'color',
|
||||
selector: '#color-pick',
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
|
||||
it('select element falls back to os_input for type (not a text input)', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'select',
|
||||
selector: '#country',
|
||||
}), true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
|
||||
it('returns reason string that includes the selector on success', () => {
|
||||
const decision = decideBrowserTypeAction(makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'text',
|
||||
selector: '#my-input',
|
||||
}), true)
|
||||
expect(decision.reason).toContain('#my-input')
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cross-function consistency
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('click + type routing consistency', () => {
|
||||
it('text input routes to clickSelector for click and setInputValue for type', () => {
|
||||
const candidate = makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'text',
|
||||
selector: '#email',
|
||||
})
|
||||
const clickD = decideBrowserAction(candidate, true)
|
||||
const typeD = decideBrowserTypeAction(candidate, true)
|
||||
|
||||
expect(clickD.route).toBe('browser_dom')
|
||||
expect(clickD.bridgeMethod).toBe('clickSelector')
|
||||
expect(typeD.route).toBe('browser_dom')
|
||||
expect(typeD.bridgeMethod).toBe('setInputValue')
|
||||
// Same selector used for both
|
||||
expect(clickD.selector).toBe(typeD.selector)
|
||||
})
|
||||
|
||||
it('checkbox routes to checkCheckbox for click but os_input for type', () => {
|
||||
const candidate = makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'checkbox',
|
||||
selector: '#agree',
|
||||
})
|
||||
const clickD = decideBrowserAction(candidate, true)
|
||||
const typeD = decideBrowserTypeAction(candidate, true)
|
||||
|
||||
expect(clickD.route).toBe('browser_dom')
|
||||
expect(clickD.bridgeMethod).toBe('checkCheckbox')
|
||||
expect(typeD.route).toBe('os_input') // Can't type into a checkbox
|
||||
})
|
||||
|
||||
it('button routes to clickSelector for click but os_input for type', () => {
|
||||
const candidate = makeCandidate({
|
||||
tag: 'button',
|
||||
role: 'button',
|
||||
selector: '#submit',
|
||||
})
|
||||
const clickD = decideBrowserAction(candidate, true)
|
||||
const typeD = decideBrowserTypeAction(candidate, true)
|
||||
|
||||
expect(clickD.route).toBe('browser_dom')
|
||||
expect(clickD.bridgeMethod).toBe('clickSelector')
|
||||
expect(typeD.route).toBe('os_input')
|
||||
})
|
||||
|
||||
it('ax candidate always routes to os_input for both click and type', () => {
|
||||
const candidate = makeCandidate({ source: 'ax', selector: '#whatever' })
|
||||
const clickD = decideBrowserAction(candidate, true)
|
||||
const typeD = decideBrowserTypeAction(candidate, true)
|
||||
|
||||
expect(clickD.route).toBe('os_input')
|
||||
expect(typeD.route).toBe('os_input')
|
||||
})
|
||||
|
||||
it('bridge-down candidate routes to os_input for both click and type', () => {
|
||||
const candidate = makeCandidate({
|
||||
tag: 'input',
|
||||
inputType: 'text',
|
||||
selector: '#email',
|
||||
})
|
||||
const clickD = decideBrowserAction(candidate, false)
|
||||
const typeD = decideBrowserTypeAction(candidate, false)
|
||||
|
||||
expect(clickD.route).toBe('os_input')
|
||||
expect(typeD.route).toBe('os_input')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Browser action router — decides whether a desktop action should go through
|
||||
* the browser-dom bridge (DOM-level precision) or OS-level input.
|
||||
*
|
||||
* Routing rules are fixed, not heuristic:
|
||||
* - chrome_dom candidate with selector + bridge available → browser_dom
|
||||
* - Everything else → os_input
|
||||
* - Bridge unavailable → os_input (graceful fallback)
|
||||
*
|
||||
* Covers: click, type/setInputValue, checkCheckbox, selectOption.
|
||||
*/
|
||||
|
||||
import type { DesktopTargetCandidate } from './desktop-grounding-types'
|
||||
|
||||
export interface BrowserActionDecision {
|
||||
/** Which execution path to use */
|
||||
route: 'browser_dom' | 'os_input'
|
||||
/** Human-readable explanation of the routing decision */
|
||||
reason: string
|
||||
/** CSS selector for browser-dom action (only when route is browser_dom) */
|
||||
selector?: string
|
||||
/** Frame ID for browser-dom action (only when route is browser_dom) */
|
||||
frameId?: number
|
||||
/** Which bridge method to use (only when route is browser_dom) */
|
||||
bridgeMethod?: 'clickSelector' | 'setInputValue' | 'checkCheckbox' | 'selectOption'
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared precondition check for browser-dom routing.
|
||||
* Returns a rejection decision if the candidate is ineligible,
|
||||
* or undefined if all preconditions pass.
|
||||
*/
|
||||
function checkBrowserDomPreconditions(
|
||||
candidate: DesktopTargetCandidate,
|
||||
bridgeAvailable: boolean,
|
||||
): BrowserActionDecision | undefined {
|
||||
if (candidate.source !== 'chrome_dom') {
|
||||
return {
|
||||
route: 'os_input',
|
||||
reason: `source is '${candidate.source}', not chrome_dom`,
|
||||
}
|
||||
}
|
||||
|
||||
if (!candidate.selector) {
|
||||
return {
|
||||
route: 'os_input',
|
||||
reason: 'chrome_dom candidate has no CSS selector for re-query',
|
||||
}
|
||||
}
|
||||
|
||||
if (!bridgeAvailable) {
|
||||
return {
|
||||
route: 'os_input',
|
||||
reason: 'browser-dom bridge is not connected, falling back to OS input',
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a click on a candidate should go through browser-dom
|
||||
* bridge or OS-level input. Also handles checkbox toggling via checkCheckbox.
|
||||
*
|
||||
* Non-left-button clicks and multi-click requests are not supported by the
|
||||
* browser-dom bridge and will always be routed to os_input.
|
||||
*/
|
||||
export function decideBrowserAction(
|
||||
candidate: DesktopTargetCandidate,
|
||||
bridgeAvailable: boolean,
|
||||
actionButton: 'left' | 'right' | 'middle' = 'left',
|
||||
clickCount: number = 1,
|
||||
): BrowserActionDecision {
|
||||
const rejection = checkBrowserDomPreconditions(candidate, bridgeAvailable)
|
||||
if (rejection)
|
||||
return rejection
|
||||
|
||||
// Right-click and multi-click are not supported by the browser-dom bridge;
|
||||
// fall through to OS input so the caller's arguments are honoured.
|
||||
if (actionButton !== 'left' || clickCount !== 1) {
|
||||
return {
|
||||
route: 'os_input',
|
||||
reason: `browser-dom click only supports left single-click; got button='${actionButton}' clickCount=${clickCount}`,
|
||||
}
|
||||
}
|
||||
|
||||
// Checkbox: route to checkCheckbox instead of generic click
|
||||
if (isCheckboxCandidate(candidate)) {
|
||||
return {
|
||||
route: 'browser_dom',
|
||||
selector: candidate.selector,
|
||||
frameId: candidate.frameId,
|
||||
bridgeMethod: 'checkCheckbox',
|
||||
reason: `chrome_dom checkbox with selector '${candidate.selector}' routed to checkCheckbox`,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
route: 'browser_dom',
|
||||
selector: candidate.selector,
|
||||
frameId: candidate.frameId,
|
||||
bridgeMethod: 'clickSelector',
|
||||
reason: `chrome_dom candidate with selector '${candidate.selector}' routed to browser-dom bridge`,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a type action should go through browser-dom setInputValue
|
||||
* or OS-level typeText.
|
||||
*
|
||||
* Only routes to browser_dom if the candidate is a text-input-like element
|
||||
* (input[text|password|email|...], textarea, or role="textbox").
|
||||
*/
|
||||
export function decideBrowserTypeAction(
|
||||
candidate: DesktopTargetCandidate,
|
||||
bridgeAvailable: boolean,
|
||||
): BrowserActionDecision {
|
||||
const rejection = checkBrowserDomPreconditions(candidate, bridgeAvailable)
|
||||
if (rejection)
|
||||
return rejection
|
||||
|
||||
if (!isTextInputCandidate(candidate)) {
|
||||
return {
|
||||
route: 'os_input',
|
||||
reason: `chrome_dom candidate tag '${candidate.tag}' is not a text input element`,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
route: 'browser_dom',
|
||||
selector: candidate.selector,
|
||||
frameId: candidate.frameId,
|
||||
bridgeMethod: 'setInputValue',
|
||||
reason: `chrome_dom text input with selector '${candidate.selector}' routed to setInputValue`,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Candidate classification helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const TEXT_INPUT_TYPES = new Set([
|
||||
'text',
|
||||
'password',
|
||||
'email',
|
||||
'search',
|
||||
'url',
|
||||
'tel',
|
||||
'number',
|
||||
])
|
||||
|
||||
/** Whether a candidate represents a text-input-like element. */
|
||||
function isTextInputCandidate(candidate: DesktopTargetCandidate): boolean {
|
||||
const tag = candidate.tag?.toLowerCase()
|
||||
if (tag === 'textarea')
|
||||
return true
|
||||
if (tag === 'input') {
|
||||
// Exclude non-text input types (checkbox, radio, file, etc.)
|
||||
const inputType = candidate.inputType?.toLowerCase() || 'text'
|
||||
return TEXT_INPUT_TYPES.has(inputType)
|
||||
}
|
||||
// NOTICE: contenteditable elements are surfaced with role="textbox" but lack
|
||||
// a native .value property, so setInputValue (which uses input/textarea value
|
||||
// setters) silently fails on them. Only route actual <input>/<textarea> here;
|
||||
// contenteditable targets will fall through to OS typing via desktop_type_text.
|
||||
return false
|
||||
}
|
||||
|
||||
/** Whether a candidate represents a checkbox or toggle. */
|
||||
function isCheckboxCandidate(candidate: DesktopTargetCandidate): boolean {
|
||||
const tag = candidate.tag?.toLowerCase()
|
||||
if (tag === 'input') {
|
||||
const inputType = candidate.inputType?.toLowerCase()
|
||||
return inputType === 'checkbox'
|
||||
}
|
||||
if (candidate.role === 'checkbox')
|
||||
return true
|
||||
return false
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { decideBrowserAction, decideBrowserTypeAction } from './browser-action-router'
|
||||
import { captureChromeSemantics, chromeElementsToTargetCandidates } from './chrome-semantic-adapter'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -115,6 +116,283 @@ describe('chromeElementsToTargetCandidates', () => {
|
||||
)
|
||||
expect(idLabel[0].label).toBe('#main-cta')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Selector building (v2)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('builds selector from element id (highest priority)', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', id: 'submit-btn', text: 'Go', rect: { x: 0, y: 0, w: 50, h: 20 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('#submit-btn')
|
||||
})
|
||||
|
||||
it('escapes special characters in id selectors', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'div', id: 'my.element:1', rect: { x: 0, y: 0, w: 50, h: 20 } }],
|
||||
windowBounds,
|
||||
)
|
||||
// dots and colons must be escaped
|
||||
expect(candidates[0].selector).toBe('#my\\.element\\:1')
|
||||
})
|
||||
|
||||
it('builds selector from name attribute (second priority)', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'input', name: 'email', rect: { x: 0, y: 0, w: 100, h: 20 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('input[name="email"]')
|
||||
})
|
||||
|
||||
it('escapes quotes in name attribute selectors', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'input', name: 'field"evil', rect: { x: 0, y: 0, w: 100, h: 20 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('input[name="field\\"evil"]')
|
||||
})
|
||||
|
||||
it('builds selector from tag+type for input elements (third priority)', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'input', type: 'submit', rect: { x: 0, y: 0, w: 80, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('input[type="submit"]')
|
||||
})
|
||||
|
||||
it('builds selector from tag+type for button elements', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', type: 'submit', rect: { x: 0, y: 0, w: 80, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('button[type="submit"]')
|
||||
})
|
||||
|
||||
it('does not use tag+type for non-input/button elements', () => {
|
||||
// A <div> with type attr should NOT get a tag[type=...] selector
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'div', type: 'custom', className: 'widget', rect: { x: 0, y: 0, w: 80, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
// Should fall through to className-based selector
|
||||
expect(candidates[0].selector).toBe('div.widget')
|
||||
})
|
||||
|
||||
it('builds selector from first className (fourth priority)', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'a', className: 'nav-link primary', rect: { x: 0, y: 0, w: 60, h: 16 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('a.nav-link')
|
||||
})
|
||||
|
||||
it('returns undefined selector when no identifying attribute exists', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'span', text: 'orphan', rect: { x: 0, y: 0, w: 40, h: 14 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBeUndefined()
|
||||
})
|
||||
|
||||
it('prefers id over name over type over className', () => {
|
||||
// Element with all attributes — id should win
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{
|
||||
tag: 'input',
|
||||
id: 'email-input',
|
||||
name: 'email',
|
||||
type: 'text',
|
||||
className: 'form-control',
|
||||
rect: { x: 0, y: 0, w: 200, h: 30 },
|
||||
}],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('#email-input')
|
||||
})
|
||||
|
||||
it('falls through to name when id is empty/whitespace', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'input', id: ' ', name: 'username', rect: { x: 0, y: 0, w: 200, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].selector).toBe('input[name="username"]')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Metadata enrichment (v2): isPageContent, enabled, inputType
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('sets isPageContent=true for all chrome_dom candidates', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[
|
||||
{ tag: 'button', text: 'A', rect: { x: 0, y: 0, w: 50, h: 20 } },
|
||||
{ tag: 'input', type: 'text', rect: { x: 0, y: 30, w: 200, h: 30 } },
|
||||
{ tag: 'a', href: '/about', text: 'About', rect: { x: 0, y: 70, w: 40, h: 16 } },
|
||||
],
|
||||
windowBounds,
|
||||
)
|
||||
for (const c of candidates) {
|
||||
expect(c.isPageContent).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('sets enabled=true for non-disabled elements', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', text: 'Active', rect: { x: 0, y: 0, w: 50, h: 20 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].enabled).toBe(true)
|
||||
})
|
||||
|
||||
it('sets enabled=false for disabled elements', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', text: 'Nope', disabled: true, rect: { x: 0, y: 0, w: 50, h: 20 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].enabled).toBe(false)
|
||||
expect(candidates[0].interactable).toBe(false)
|
||||
})
|
||||
|
||||
it('carries inputType from element type attribute', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'input', type: 'password', rect: { x: 0, y: 0, w: 200, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].inputType).toBe('password')
|
||||
})
|
||||
|
||||
it('carries href for link elements', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'a', href: 'https://example.com', text: 'Link', rect: { x: 0, y: 0, w: 40, h: 16 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].href).toBe('https://example.com')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Frame ID propagation (v2)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('uses default frameId=0 when not specified', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', text: 'Main', rect: { x: 0, y: 0, w: 50, h: 20 } }],
|
||||
windowBounds,
|
||||
)
|
||||
expect(candidates[0].frameId).toBe(0)
|
||||
})
|
||||
|
||||
it('uses explicit frameId parameter', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', text: 'Iframe', rect: { x: 0, y: 0, w: 50, h: 20 } }],
|
||||
windowBounds,
|
||||
88, // chrome height
|
||||
5, // frameId
|
||||
)
|
||||
expect(candidates[0].frameId).toBe(5)
|
||||
})
|
||||
|
||||
it('reads per-element _frameId from tagged elements (extension bridge)', () => {
|
||||
// The extension bridge tags each element with _frameId
|
||||
const taggedEl = {
|
||||
tag: 'input',
|
||||
type: 'text',
|
||||
rect: { x: 0, y: 0, w: 200, h: 30 },
|
||||
_frameId: 3,
|
||||
} as any
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[taggedEl],
|
||||
windowBounds,
|
||||
88, // chrome height
|
||||
0, // default frameId param = 0
|
||||
)
|
||||
// Per-element _frameId should override the function-level param
|
||||
expect(candidates[0].frameId).toBe(3)
|
||||
})
|
||||
|
||||
it('falls back to function-level frameId when _frameId is absent', () => {
|
||||
const el = {
|
||||
tag: 'button',
|
||||
text: 'No tag',
|
||||
rect: { x: 0, y: 0, w: 50, h: 20 },
|
||||
// no _frameId
|
||||
}
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[el],
|
||||
windowBounds,
|
||||
88,
|
||||
7,
|
||||
)
|
||||
expect(candidates[0].frameId).toBe(7)
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// End-to-end routing scenario: selector → router → decision
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('candidate with id goes through full routing as browser_dom click', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', id: 'login-btn', text: 'Login', rect: { x: 0, y: 0, w: 80, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
// Assign an id like the grounding layer would
|
||||
candidates[0].id = 't_0'
|
||||
|
||||
const decision = decideBrowserAction(candidates[0], true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('clickSelector')
|
||||
expect(decision.selector).toBe('#login-btn')
|
||||
})
|
||||
|
||||
it('candidate without identifiers routes to os_input', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'span', text: 'plain text', rect: { x: 0, y: 0, w: 60, h: 14 } }],
|
||||
windowBounds,
|
||||
)
|
||||
candidates[0].id = 't_0'
|
||||
|
||||
const decision = decideBrowserAction(candidates[0], true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('no CSS selector')
|
||||
})
|
||||
|
||||
it('checkbox candidate goes through routing as checkCheckbox', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'input', type: 'checkbox', id: 'agree', rect: { x: 0, y: 0, w: 16, h: 16 } }],
|
||||
windowBounds,
|
||||
)
|
||||
candidates[0].id = 't_0'
|
||||
|
||||
const decision = decideBrowserAction(candidates[0], true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('checkCheckbox')
|
||||
})
|
||||
|
||||
it('text input candidate goes through type routing as setInputValue', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'input', type: 'email', name: 'user-email', rect: { x: 0, y: 0, w: 200, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
candidates[0].id = 't_0'
|
||||
|
||||
const decision = decideBrowserTypeAction(candidates[0], true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
expect(decision.bridgeMethod).toBe('setInputValue')
|
||||
expect(decision.selector).toBe('input[name="user-email"]')
|
||||
})
|
||||
|
||||
it('non-text-input candidate falls back to os_input for type action', () => {
|
||||
const candidates = chromeElementsToTargetCandidates(
|
||||
[{ tag: 'button', id: 'send', text: 'Send', rect: { x: 0, y: 0, w: 80, h: 30 } }],
|
||||
windowBounds,
|
||||
)
|
||||
candidates[0].id = 't_0'
|
||||
|
||||
const decision = decideBrowserTypeAction(candidates[0], true)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('not a text input')
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -151,185 +429,6 @@ describe('captureChromeSemantics', () => {
|
||||
expect(result!.interactiveElements).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('falls back to CDP when extension capture returns no interactive elements', async () => {
|
||||
const mockExtension = {
|
||||
getStatus: () => ({ connected: true, enabled: true, host: 'localhost', port: 8080, pendingRequests: 0 }),
|
||||
readAllFramesDom: vi.fn().mockResolvedValue([
|
||||
{
|
||||
frameId: 0,
|
||||
result: {
|
||||
url: 'https://example.com',
|
||||
title: 'Example',
|
||||
interactiveElements: [],
|
||||
},
|
||||
},
|
||||
]),
|
||||
}
|
||||
const mockCdp = {
|
||||
getStatus: vi.fn().mockReturnValue({
|
||||
connected: true,
|
||||
pageUrl: 'https://example.com',
|
||||
pageTitle: 'Example',
|
||||
}),
|
||||
collectInteractiveElements: vi.fn().mockResolvedValue([
|
||||
{ tag: 'button', text: 'Fallback CTA', rect: { x: 0, y: 0, w: 50, h: 20 } },
|
||||
]),
|
||||
}
|
||||
|
||||
const result = await captureChromeSemantics(mockExtension as any, mockCdp as any)
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.source).toBe('cdp')
|
||||
expect(result!.interactiveElements).toHaveLength(1)
|
||||
expect(result!.interactiveElements[0].text).toBe('Fallback CTA')
|
||||
})
|
||||
|
||||
it('unwraps extension frame payloads nested under result.data', async () => {
|
||||
const mockExtension = {
|
||||
getStatus: () => ({ connected: true, enabled: true, host: 'localhost', port: 8080, pendingRequests: 0 }),
|
||||
readAllFramesDom: vi.fn().mockResolvedValue([
|
||||
{
|
||||
frameId: 0,
|
||||
result: {
|
||||
data: {
|
||||
url: 'https://nested.example.com',
|
||||
title: 'Nested Example',
|
||||
interactiveElements: [
|
||||
{ tag: 'button', text: 'Nested click', rect: { x: 0, y: 0, w: 50, h: 20 } },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
const result = await captureChromeSemantics(mockExtension as any, undefined)
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.pageUrl).toBe('https://nested.example.com')
|
||||
expect(result!.pageTitle).toBe('Nested Example')
|
||||
expect(result!.interactiveElements).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('applies iframe offsets before returning extension frame elements', async () => {
|
||||
const mockExtension = {
|
||||
getStatus: () => ({ connected: true, enabled: true, host: 'localhost', port: 8080, pendingRequests: 0 }),
|
||||
getAllFrames: vi.fn().mockResolvedValue([
|
||||
{ frameId: 0, parentFrameId: -1 },
|
||||
{ frameId: 7, parentFrameId: 0 },
|
||||
]),
|
||||
readAllFramesDom: vi.fn().mockResolvedValue([
|
||||
{
|
||||
frameId: 0,
|
||||
result: {
|
||||
url: 'https://example.com',
|
||||
title: 'Example',
|
||||
interactiveElements: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: 7,
|
||||
result: {
|
||||
frameRect: { x: 120, y: 80, w: 640, h: 480 },
|
||||
interactiveElements: [
|
||||
{ tag: 'button', text: 'Iframe CTA', rect: { x: 10, y: 20, w: 50, h: 20 } },
|
||||
],
|
||||
},
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
const result = await captureChromeSemantics(mockExtension as any, undefined)
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.interactiveElements).toHaveLength(1)
|
||||
expect(result!.interactiveElements[0].rect).toEqual({
|
||||
x: 130,
|
||||
y: 100,
|
||||
w: 50,
|
||||
h: 20,
|
||||
})
|
||||
})
|
||||
|
||||
it('skips subframe elements when iframe offsets are unavailable', async () => {
|
||||
const mockExtension = {
|
||||
getStatus: () => ({ connected: true, enabled: true, host: 'localhost', port: 8080, pendingRequests: 0 }),
|
||||
getAllFrames: vi.fn().mockResolvedValue([
|
||||
{ frameId: 0, parentFrameId: -1 },
|
||||
{ frameId: 9, parentFrameId: 0 },
|
||||
]),
|
||||
readAllFramesDom: vi.fn().mockResolvedValue([
|
||||
{
|
||||
frameId: 0,
|
||||
result: {
|
||||
url: 'https://example.com',
|
||||
title: 'Example',
|
||||
interactiveElements: [
|
||||
{ tag: 'button', text: 'Root CTA', rect: { x: 0, y: 0, w: 20, h: 20 } },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: 9,
|
||||
result: {
|
||||
interactiveElements: [
|
||||
{ tag: 'button', text: 'Iframe CTA', rect: { x: 10, y: 20, w: 50, h: 20 } },
|
||||
],
|
||||
},
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
const result = await captureChromeSemantics(mockExtension as any, undefined)
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.interactiveElements).toHaveLength(1)
|
||||
expect(result!.interactiveElements[0].text).toBe('Root CTA')
|
||||
})
|
||||
|
||||
it('resolves nested iframe offsets even when frame results arrive out of order', async () => {
|
||||
const mockExtension = {
|
||||
getStatus: () => ({ connected: true, enabled: true, host: 'localhost', port: 8080, pendingRequests: 0 }),
|
||||
getAllFrames: vi.fn().mockResolvedValue([
|
||||
{ frameId: 0, parentFrameId: -1 },
|
||||
{ frameId: 7, parentFrameId: 0 },
|
||||
{ frameId: 12, parentFrameId: 7 },
|
||||
]),
|
||||
readAllFramesDom: vi.fn().mockResolvedValue([
|
||||
{
|
||||
frameId: 12,
|
||||
result: {
|
||||
frameRect: { x: 15, y: 25, w: 320, h: 200 },
|
||||
interactiveElements: [
|
||||
{ tag: 'button', text: 'Nested CTA', rect: { x: 3, y: 4, w: 40, h: 20 } },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: 0,
|
||||
result: {
|
||||
url: 'https://example.com',
|
||||
title: 'Example',
|
||||
interactiveElements: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: 7,
|
||||
result: {
|
||||
frameRect: { x: 120, y: 80, w: 640, h: 480 },
|
||||
interactiveElements: [],
|
||||
},
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
const result = await captureChromeSemantics(mockExtension as any, undefined)
|
||||
expect(result).not.toBeNull()
|
||||
expect(result!.interactiveElements).toHaveLength(1)
|
||||
expect(result!.interactiveElements[0].rect).toEqual({
|
||||
x: 138,
|
||||
y: 109,
|
||||
w: 40,
|
||||
h: 20,
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to CDP when extension is disconnected', async () => {
|
||||
const mockExtension = {
|
||||
getStatus: () => ({ connected: false, enabled: true, host: 'localhost', port: 8080, pendingRequests: 0 }),
|
||||
|
||||
@@ -18,7 +18,6 @@ import type {
|
||||
} from './desktop-grounding-types'
|
||||
import type {
|
||||
Bounds,
|
||||
BrowserDomFrameDom,
|
||||
BrowserDomInteractiveElement,
|
||||
} from './types'
|
||||
|
||||
@@ -33,6 +32,11 @@ import type {
|
||||
*/
|
||||
const CHROME_CHROME_HEIGHT_PX = 88
|
||||
|
||||
// Pre-compiled regex for selector building (module scope per eslint e18e/prefer-static-regex)
|
||||
const RE_DOUBLE_QUOTE = /"/g
|
||||
const RE_WHITESPACE_SPLIT = /\s+/
|
||||
const RE_CSS_ESCAPE = /[^\w-]/g
|
||||
|
||||
/**
|
||||
* Capture Chrome semantic data from the active tab.
|
||||
*
|
||||
@@ -53,10 +57,7 @@ export async function captureChromeSemantics(
|
||||
try {
|
||||
const status = extensionBridge.getStatus()
|
||||
if (status.connected) {
|
||||
const extensionSnapshot = await captureViaExtension(extensionBridge)
|
||||
if (extensionSnapshot.interactiveElements.length > 0 || !cdpBridge?.getStatus().connected) {
|
||||
return extensionSnapshot
|
||||
}
|
||||
return await captureViaExtension(extensionBridge)
|
||||
}
|
||||
}
|
||||
catch {
|
||||
@@ -95,6 +96,7 @@ export function chromeElementsToTargetCandidates(
|
||||
elements: BrowserDomInteractiveElement[],
|
||||
windowBounds: Bounds,
|
||||
chromeHeightPx: number = CHROME_CHROME_HEIGHT_PX,
|
||||
frameId: number = 0,
|
||||
): DesktopTargetCandidate[] {
|
||||
const candidates: DesktopTargetCandidate[] = []
|
||||
const viewportOffsetX = windowBounds.x
|
||||
@@ -105,6 +107,10 @@ export function chromeElementsToTargetCandidates(
|
||||
continue
|
||||
}
|
||||
|
||||
// Read per-element frame ID if tagged by captureViaExtension,
|
||||
// otherwise fall back to the function parameter
|
||||
const elFrameId = (el as Record<string, unknown>)._frameId as number | undefined
|
||||
|
||||
// Convert page-relative rect to screen-absolute bounds
|
||||
const bounds: Bounds = {
|
||||
x: viewportOffsetX + el.rect.x,
|
||||
@@ -124,6 +130,7 @@ export function chromeElementsToTargetCandidates(
|
||||
const label = buildLabel(el)
|
||||
const role = el.role || el.tag || 'element'
|
||||
const confidence = computeElementConfidence(el)
|
||||
const selector = buildSelector(el)
|
||||
|
||||
candidates.push({
|
||||
id: '', // Will be assigned by the grounding layer
|
||||
@@ -137,6 +144,10 @@ export function chromeElementsToTargetCandidates(
|
||||
tag: el.tag,
|
||||
href: el.href,
|
||||
inputType: el.type,
|
||||
selector,
|
||||
frameId: elFrameId ?? frameId,
|
||||
isPageContent: true, // All chrome_dom candidates are page content by definition
|
||||
enabled: !el.disabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -147,107 +158,6 @@ export function chromeElementsToTargetCandidates(
|
||||
// Internal helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function toRecord(value: unknown): Record<string, unknown> | undefined {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value))
|
||||
return undefined
|
||||
|
||||
return value as Record<string, unknown>
|
||||
}
|
||||
|
||||
function toFiniteNumber(value: unknown): number | undefined {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined
|
||||
}
|
||||
|
||||
function getExtensionFramePayload(result: Record<string, unknown>) {
|
||||
return toRecord(result.data) ?? result
|
||||
}
|
||||
|
||||
function getFrameRect(payload: Record<string, unknown>): BrowserDomFrameDom['frameRect'] | undefined {
|
||||
const rect = toRecord(payload.frameRect)
|
||||
if (!rect)
|
||||
return undefined
|
||||
|
||||
const x = toFiniteNumber(rect.x)
|
||||
const y = toFiniteNumber(rect.y)
|
||||
const w = toFiniteNumber(rect.w)
|
||||
const h = toFiniteNumber(rect.h)
|
||||
if (x === undefined || y === undefined || w === undefined || h === undefined)
|
||||
return undefined
|
||||
|
||||
return { x, y, w, h }
|
||||
}
|
||||
|
||||
function getFrameParentId(frame: Record<string, unknown>): number | undefined {
|
||||
return toFiniteNumber(frame.parentFrameId)
|
||||
}
|
||||
|
||||
function offsetInteractiveElement(
|
||||
element: BrowserDomInteractiveElement,
|
||||
offset: { x: number, y: number },
|
||||
): BrowserDomInteractiveElement {
|
||||
return {
|
||||
...element,
|
||||
rect: element.rect
|
||||
? {
|
||||
...element.rect,
|
||||
x: element.rect.x + offset.x,
|
||||
y: element.rect.y + offset.y,
|
||||
}
|
||||
: element.rect,
|
||||
center: element.center
|
||||
? {
|
||||
x: element.center.x + offset.x,
|
||||
y: element.center.y + offset.y,
|
||||
}
|
||||
: element.center,
|
||||
}
|
||||
}
|
||||
|
||||
function resolveFrameOffset(
|
||||
frameId: number,
|
||||
parentIds: Map<number, number | undefined>,
|
||||
payloads: Map<number, Record<string, unknown>>,
|
||||
cache: Map<number, { x: number, y: number } | null>,
|
||||
visiting: Set<number> = new Set(),
|
||||
): { x: number, y: number } | null {
|
||||
if (cache.has(frameId))
|
||||
return cache.get(frameId) ?? null
|
||||
|
||||
if (frameId === 0) {
|
||||
const rootOffset = { x: 0, y: 0 }
|
||||
cache.set(frameId, rootOffset)
|
||||
return rootOffset
|
||||
}
|
||||
|
||||
if (visiting.has(frameId)) {
|
||||
return null
|
||||
}
|
||||
|
||||
visiting.add(frameId)
|
||||
|
||||
const payload = payloads.get(frameId)
|
||||
const frameRect = payload ? getFrameRect(payload) : undefined
|
||||
const parentFrameId = parentIds.get(frameId)
|
||||
if (!frameRect || parentFrameId === undefined) {
|
||||
visiting.delete(frameId)
|
||||
return null
|
||||
}
|
||||
|
||||
const parentOffset = resolveFrameOffset(parentFrameId, parentIds, payloads, cache, visiting)
|
||||
if (!parentOffset) {
|
||||
visiting.delete(frameId)
|
||||
return null
|
||||
}
|
||||
|
||||
const resolvedOffset = {
|
||||
x: parentOffset.x + frameRect.x,
|
||||
y: parentOffset.y + frameRect.y,
|
||||
}
|
||||
cache.set(frameId, resolvedOffset)
|
||||
visiting.delete(frameId)
|
||||
return resolvedOffset
|
||||
}
|
||||
|
||||
async function captureViaExtension(
|
||||
bridge: BrowserDomExtensionBridge,
|
||||
): Promise<ChromeSemanticSnapshot> {
|
||||
@@ -255,67 +165,30 @@ async function captureViaExtension(
|
||||
includeText: false,
|
||||
maxElements: 150,
|
||||
})
|
||||
const frameTree = typeof bridge.getAllFrames === 'function'
|
||||
? await bridge.getAllFrames().catch(() => [])
|
||||
: []
|
||||
|
||||
// Merge interactive elements from all frames
|
||||
const allElements: BrowserDomInteractiveElement[] = []
|
||||
// Merge interactive elements from all frames, preserving frame identity
|
||||
const allElements: Array<BrowserDomInteractiveElement & { _frameId?: number }> = []
|
||||
let pageUrl = ''
|
||||
let pageTitle = ''
|
||||
const payloadsByFrameId = new Map<number, Record<string, unknown>>()
|
||||
const parentIdsByFrameId = new Map<number, number | undefined>()
|
||||
const resolvedOffsets = new Map<number, { x: number, y: number } | null>()
|
||||
|
||||
for (const frame of frameTree) {
|
||||
const frameRecord = toRecord(frame)
|
||||
if (!frameRecord)
|
||||
continue
|
||||
|
||||
const frameId = toFiniteNumber(frameRecord.frameId)
|
||||
if (frameId === undefined)
|
||||
continue
|
||||
|
||||
parentIdsByFrameId.set(frameId, getFrameParentId(frameRecord))
|
||||
}
|
||||
|
||||
for (const frame of frames) {
|
||||
const dom = frame.result as Record<string, unknown> | undefined
|
||||
if (!dom)
|
||||
continue
|
||||
|
||||
const payload = getExtensionFramePayload(dom)
|
||||
payloadsByFrameId.set(frame.frameId, payload)
|
||||
|
||||
if (frame.frameId === 0) {
|
||||
pageUrl = (payload.url as string) || ''
|
||||
pageTitle = (payload.title as string) || ''
|
||||
pageUrl = (dom.url as string) || ''
|
||||
pageTitle = (dom.title as string) || ''
|
||||
}
|
||||
}
|
||||
|
||||
for (const frame of frames) {
|
||||
const payload = payloadsByFrameId.get(frame.frameId)
|
||||
if (!payload)
|
||||
continue
|
||||
|
||||
const rawElements = payload.interactiveElements
|
||||
const rawElements = dom.interactiveElements
|
||||
?? (dom.data && typeof dom.data === 'object' && (dom.data as Record<string, unknown>).interactiveElements)
|
||||
const elements = rawElements as BrowserDomInteractiveElement[] | undefined
|
||||
if (elements) {
|
||||
const offset = resolveFrameOffset(
|
||||
frame.frameId,
|
||||
parentIdsByFrameId,
|
||||
payloadsByFrameId,
|
||||
resolvedOffsets,
|
||||
)
|
||||
|
||||
if (frame.frameId !== 0 && !offset) {
|
||||
continue
|
||||
// Tag each element with its frame ID for downstream routing
|
||||
for (const el of elements) {
|
||||
allElements.push({ ...el, _frameId: frame.frameId })
|
||||
}
|
||||
|
||||
const normalizedElements = offset
|
||||
? elements.map(element => offsetInteractiveElement(element, offset))
|
||||
: elements
|
||||
allElements.push(...normalizedElements)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -359,6 +232,52 @@ async function captureViaCdp(bridge: CdpBridge): Promise<ChromeSemanticSnapshot>
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a best-effort CSS selector for re-querying the element via the
|
||||
* browser-dom bridge. Used by the browser action router for DOM-level
|
||||
* click precision instead of OS coordinate input.
|
||||
*
|
||||
* Priority: #id > [name] > tag[type] > tag.className > tag
|
||||
*/
|
||||
function buildSelector(el: BrowserDomInteractiveElement): string | undefined {
|
||||
// Unique id — best
|
||||
if (el.id && el.id.trim()) {
|
||||
return `#${cssEscape(el.id.trim())}`
|
||||
}
|
||||
|
||||
const tag = el.tag?.toLowerCase() || '*'
|
||||
|
||||
// Name attribute — common for form inputs
|
||||
if (el.name && el.name.trim()) {
|
||||
return `${tag}[name="${el.name.trim().replace(RE_DOUBLE_QUOTE, '\\"')}"]`
|
||||
}
|
||||
|
||||
// Tag + type — useful for input[type="submit"] etc.
|
||||
if (el.type && el.type.trim() && (tag === 'input' || tag === 'button')) {
|
||||
return `${tag}[type="${el.type.trim().replace(RE_DOUBLE_QUOTE, '\\"')}"]`
|
||||
}
|
||||
|
||||
// Tag + first className — fallback
|
||||
if (el.className && el.className.trim()) {
|
||||
const firstClass = el.className.trim().split(RE_WHITESPACE_SPLIT)[0]
|
||||
if (firstClass) {
|
||||
return `${tag}.${cssEscape(firstClass)}`
|
||||
}
|
||||
}
|
||||
|
||||
// Tag alone is too generic to be useful
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal CSS identifier escape for Node.js (CSS.escape is browser-only).
|
||||
* Escapes characters that are invalid in CSS identifiers per the spec.
|
||||
* Sufficient for id/class name escaping in selector construction.
|
||||
*/
|
||||
function cssEscape(value: string): string {
|
||||
return value.replace(RE_CSS_ESCAPE, ch => `\\${ch}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a human-readable label from element attributes.
|
||||
* Priority: text > placeholder > name > id > href > tag.
|
||||
|
||||
@@ -79,6 +79,10 @@ export interface DesktopTargetCandidate {
|
||||
inputType?: string
|
||||
/** CSS selector for re-querying (best-effort) */
|
||||
selector?: string
|
||||
/** Frame ID within the Chrome page (0 = main frame) */
|
||||
frameId?: number
|
||||
/** Whether candidate is in page content area (true for all chrome_dom candidates) */
|
||||
isPageContent?: boolean
|
||||
|
||||
// ---- AX extras ----
|
||||
/** AX tree UID for `findAXNodeByUid` lookup */
|
||||
|
||||
@@ -26,9 +26,7 @@ import type {
|
||||
} from './types'
|
||||
|
||||
import { captureAXTree } from './accessibility'
|
||||
import { appNamesMatch } from './app-aliases'
|
||||
import { captureChromeSemantics, chromeElementsToTargetCandidates } from './chrome-semantic-adapter'
|
||||
import { TARGET_SOURCE_PRIORITY } from './desktop-grounding-types'
|
||||
import { boundsIoU } from './snap-resolver'
|
||||
|
||||
/**
|
||||
@@ -66,6 +64,7 @@ export async function captureDesktopGrounding(params: {
|
||||
cdpBridge?: CdpBridge
|
||||
}): Promise<DesktopGroundingSnapshot> {
|
||||
const { config, executor, input, extensionBridge, cdpBridge } = params
|
||||
const assemblyStart = Date.now()
|
||||
|
||||
// Phase 1: Parallel capture of all observation sources
|
||||
const [screenshotResult, windowsResult, axResult] = await Promise.allSettled([
|
||||
@@ -99,7 +98,6 @@ export async function captureDesktopGrounding(params: {
|
||||
|
||||
// Phase 4: Compute staleness
|
||||
const now = Date.now()
|
||||
const capturedAt = new Date(now).toISOString()
|
||||
const staleFlags = computeStaleness({
|
||||
screenshot,
|
||||
axSnapshot,
|
||||
@@ -112,9 +110,8 @@ export async function captureDesktopGrounding(params: {
|
||||
|
||||
return {
|
||||
snapshotId,
|
||||
capturedAt,
|
||||
capturedAt: new Date(assemblyStart).toISOString(),
|
||||
foregroundApp,
|
||||
foregroundWindowTitle: windowObs.frontmostWindowTitle,
|
||||
windows: windowObs.windows,
|
||||
screenshot,
|
||||
axSnapshot,
|
||||
@@ -176,10 +173,8 @@ export function buildTargetCandidates(params: {
|
||||
// Sort: chrome_dom first, then ax, then by confidence desc
|
||||
merged.sort((a, b) => {
|
||||
if (a.source !== b.source) {
|
||||
const aPriority = TARGET_SOURCE_PRIORITY.indexOf(a.source)
|
||||
const bPriority = TARGET_SOURCE_PRIORITY.indexOf(b.source)
|
||||
return (aPriority === -1 ? TARGET_SOURCE_PRIORITY.length : aPriority)
|
||||
- (bPriority === -1 ? TARGET_SOURCE_PRIORITY.length : bPriority)
|
||||
const sourceOrder: Record<string, number> = { chrome_dom: 0, ax: 1, vision: 2, raw: 3 }
|
||||
return (sourceOrder[a.source] ?? 3) - (sourceOrder[b.source] ?? 3)
|
||||
}
|
||||
return b.confidence - a.confidence
|
||||
})
|
||||
@@ -327,29 +322,18 @@ function findChromeWindowBounds(
|
||||
observation: WindowObservation,
|
||||
foregroundApp: string,
|
||||
): Bounds | undefined {
|
||||
if (!isChromeApp(foregroundApp))
|
||||
return undefined
|
||||
|
||||
const chromeWindows = observation.windows.filter(window =>
|
||||
window.bounds
|
||||
&& window.isOnScreen !== false
|
||||
&& isChromeApp(window.appName),
|
||||
const normalizedFg = foregroundApp.trim().toLowerCase().replace(APP_SUFFIX_RE, '')
|
||||
// Prefer exact match on the foreground app name
|
||||
const exactMatch = observation.windows.find(w =>
|
||||
w.appName.trim().toLowerCase().replace(APP_SUFFIX_RE, '') === normalizedFg && w.bounds,
|
||||
)
|
||||
if (chromeWindows.length === 0)
|
||||
return undefined
|
||||
|
||||
const foregroundChromeWindows = chromeWindows.filter(window => appNamesMatch(window.appName, foregroundApp))
|
||||
const preferredWindows = foregroundChromeWindows.length > 0 ? foregroundChromeWindows : chromeWindows
|
||||
|
||||
const frontmostTitle = observation.frontmostWindowTitle?.trim()
|
||||
if (frontmostTitle) {
|
||||
const frontmostWindow = preferredWindows.find(window => window.title?.trim() === frontmostTitle)
|
||||
if (frontmostWindow?.bounds) {
|
||||
return frontmostWindow.bounds
|
||||
}
|
||||
}
|
||||
|
||||
return preferredWindows[0]?.bounds
|
||||
if (exactMatch?.bounds)
|
||||
return exactMatch.bounds
|
||||
// Fallback: any Chrome-like window
|
||||
const chromeWindow = observation.windows.find(w =>
|
||||
isChromeApp(w.appName) && w.bounds,
|
||||
)
|
||||
return chromeWindow?.bounds
|
||||
}
|
||||
|
||||
function computeStaleness(params: {
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
import type { ComputerUseServerRuntime } from './runtime'
|
||||
|
||||
import { appNamesMatch, normalizeConfiguredAppAction } from '../app-aliases'
|
||||
import { decideBrowserTypeAction } from '../browser-action-router'
|
||||
import { DESKTOP_CLICK_SNAPSHOT_MAX_AGE_MS } from '../desktop-grounding-types'
|
||||
import { evaluateActionPolicy } from '../policy'
|
||||
import { getRuntimePreflight } from '../preflight'
|
||||
@@ -466,10 +467,70 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
|
||||
throw new Error(`Preparatory click at (${normalizedAction.input.x}, ${normalizedAction.input.y}) failed before typing: ${msg}`)
|
||||
}
|
||||
}
|
||||
const result = await runtime.executor.typeText(normalizedAction.input)
|
||||
backendResult = {
|
||||
...backendResult,
|
||||
...result,
|
||||
|
||||
// Browser-dom type routing: if the last clicked grounding candidate
|
||||
// is a chrome_dom text input, use setInputValue for DOM precision.
|
||||
// NOTICE: skip this path when explicit coordinates are provided.
|
||||
// Coordinates mean the caller has targeted a specific screen position
|
||||
// (possibly in a different app/window), so using lastClickedCandidateId
|
||||
// would write into a stale Chrome selector instead of the current target.
|
||||
const hasExplicitCoords = typeof normalizedAction.input.x === 'number' && typeof normalizedAction.input.y === 'number'
|
||||
let usedBrowserDom = false
|
||||
const runState = runtime.stateManager.getState()
|
||||
const lastSnapshot = runState.lastGroundingSnapshot
|
||||
const lastClickedId = runState.lastClickedCandidateId
|
||||
if (!hasExplicitCoords && lastClickedId && lastSnapshot) {
|
||||
const lastCandidate = lastSnapshot.targetCandidates.find(
|
||||
c => c.id === lastClickedId,
|
||||
)
|
||||
if (lastCandidate) {
|
||||
const bridgeConnected = runtime.browserDomBridge?.getStatus().connected ?? false
|
||||
const typeDecision = decideBrowserTypeAction(lastCandidate, bridgeConnected)
|
||||
if (typeDecision.route === 'browser_dom' && typeDecision.selector) {
|
||||
try {
|
||||
const frameResults = await runtime.browserDomBridge!.setInputValue({
|
||||
selector: typeDecision.selector,
|
||||
value: normalizedAction.input.text,
|
||||
simulateKeystrokes: false,
|
||||
blur: !normalizedAction.input.pressEnter,
|
||||
frameIds: typeDecision.frameId !== undefined
|
||||
? [typeDecision.frameId]
|
||||
: undefined,
|
||||
})
|
||||
// NOTICE: bridge resolve ≠ DOM success. Frame results carry
|
||||
// per-frame { success, error } — if none succeeded the
|
||||
// selector/frame was stale and we must fall back to OS typeText.
|
||||
const anySucceeded = Array.isArray(frameResults) && frameResults.some(
|
||||
fr => (fr.result as Record<string, unknown>)?.success === true,
|
||||
)
|
||||
if (!anySucceeded) {
|
||||
throw new Error('setInputValue: no frame reported success')
|
||||
}
|
||||
usedBrowserDom = true
|
||||
backendResult.browserDomRoute = {
|
||||
method: 'setInputValue',
|
||||
selector: typeDecision.selector,
|
||||
reason: typeDecision.reason,
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Fallback to OS typeText below
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!usedBrowserDom) {
|
||||
const result = await runtime.executor.typeText(normalizedAction.input)
|
||||
backendResult = {
|
||||
...backendResult,
|
||||
...result,
|
||||
}
|
||||
}
|
||||
|
||||
// Handle pressEnter even when browser-dom was used
|
||||
if (usedBrowserDom && normalizedAction.input.pressEnter) {
|
||||
await runtime.executor.pressKeys({ keys: ['Return'] })
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import type {
|
||||
TargetSource,
|
||||
} from '../desktop-grounding-types'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { RunStateManager } from '../state'
|
||||
|
||||
@@ -68,7 +68,8 @@ describe('runStateManager grounding state', () => {
|
||||
source: 'chrome_dom' as TargetSource,
|
||||
confidence: 0.95,
|
||||
path: [{ x: 140, y: 215, delayMs: 0 }],
|
||||
}, 't_0')
|
||||
}, 't_test')
|
||||
sm.recordClickedCandidate('t_0')
|
||||
|
||||
expect(sm.getState().lastClickedCandidateId).toBe('t_0')
|
||||
|
||||
@@ -88,7 +89,8 @@ describe('runStateManager grounding state', () => {
|
||||
confidence: 0.9,
|
||||
path: [{ x: 330, y: 213, delayMs: 0 }],
|
||||
}
|
||||
sm.updatePointerIntent(intent, 't_1')
|
||||
sm.updatePointerIntent(intent, 't_test')
|
||||
sm.recordClickedCandidate('t_1')
|
||||
|
||||
const state = sm.getState()
|
||||
expect(state.lastPointerIntent).toBe(intent)
|
||||
@@ -106,7 +108,8 @@ describe('runStateManager grounding state', () => {
|
||||
source: 'chrome_dom' as TargetSource,
|
||||
confidence: 0.95,
|
||||
path: [{ x: 140, y: 215, delayMs: 0 }],
|
||||
}, 't_0')
|
||||
}, 't_test')
|
||||
sm.recordClickedCandidate('t_0')
|
||||
|
||||
sm.clearGroundingState()
|
||||
|
||||
@@ -135,7 +138,8 @@ describe('desktop_click_target preconditions via RunStateManager', () => {
|
||||
source: 'chrome_dom' as TargetSource,
|
||||
confidence: 0.95,
|
||||
path: [{ x: 140, y: 215, delayMs: 0 }],
|
||||
}, 't_0')
|
||||
}, 't_test')
|
||||
sm.recordClickedCandidate('t_0')
|
||||
|
||||
expect(sm.getState().lastClickedCandidateId === 't_0').toBe(true)
|
||||
})
|
||||
@@ -154,7 +158,8 @@ describe('desktop_click_target preconditions via RunStateManager', () => {
|
||||
source: 'chrome_dom' as TargetSource,
|
||||
confidence: 0.95,
|
||||
path: [{ x: 140, y: 215, delayMs: 0 }],
|
||||
}, 't_0')
|
||||
}, 't_test')
|
||||
sm.recordClickedCandidate('t_0')
|
||||
|
||||
expect(sm.getState().lastClickedCandidateId === 't_1').toBe(false)
|
||||
})
|
||||
@@ -170,7 +175,8 @@ describe('desktop_click_target preconditions via RunStateManager', () => {
|
||||
source: 'chrome_dom' as TargetSource,
|
||||
confidence: 0.95,
|
||||
path: [{ x: 140, y: 215, delayMs: 0 }],
|
||||
}, 't_0')
|
||||
}, 't_test')
|
||||
sm.recordClickedCandidate('t_0')
|
||||
|
||||
// Re-observe resets clicked candidate
|
||||
sm.updateGroundingSnapshot(makeSnapshot())
|
||||
@@ -235,7 +241,8 @@ describe('overlay polling contract: desktop_get_state exposes grounding data', (
|
||||
source: 'chrome_dom' as TargetSource,
|
||||
confidence: 0.95,
|
||||
path: [{ x: 140, y: 215, delayMs: 0 }],
|
||||
}, 't_0')
|
||||
}, 't_test')
|
||||
sm.recordClickedCandidate('t_0')
|
||||
|
||||
const state = sm.getState()
|
||||
expect(state.lastPointerIntent).toBeDefined()
|
||||
@@ -254,3 +261,589 @@ describe('overlay polling contract: desktop_get_state exposes grounding data', (
|
||||
expect(state.lastClickedCandidateId).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// desktop_click_target handler integration tests
|
||||
//
|
||||
// These simulate the handler execution path from register-desktop-grounding.ts
|
||||
// with mocked runtime dependencies to verify that routing decisions translate
|
||||
// into real bridge/executor calls and correct response text.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('desktop_click_target handler integration', () => {
|
||||
// Replicates the handler logic from register-desktop-grounding.ts
|
||||
// into a testable function. Uses the same imports the handler uses.
|
||||
async function simulateClickTargetHandler(params: {
|
||||
stateManager: RunStateManager
|
||||
candidateId: string
|
||||
button?: string
|
||||
clickCount?: number
|
||||
browserDomBridge: {
|
||||
getStatus: () => { connected: boolean }
|
||||
clickSelector: (args: { selector: string, frameIds?: number[] }) => Promise<void>
|
||||
checkCheckbox: (args: { selector: string, frameIds?: number[] }) => Promise<void>
|
||||
}
|
||||
executor: {
|
||||
click: (args: Record<string, unknown>) => Promise<Record<string, unknown>>
|
||||
}
|
||||
}) {
|
||||
const { stateManager, candidateId, button, clickCount, browserDomBridge, executor } = params
|
||||
const { decideBrowserAction } = await import('../browser-action-router')
|
||||
const { resolveSnapByCandidate } = await import('../snap-resolver')
|
||||
|
||||
const state = stateManager.getState()
|
||||
|
||||
if (!state.lastGroundingSnapshot) {
|
||||
return { isError: true, text: 'No snapshot' }
|
||||
}
|
||||
|
||||
const snapshot = state.lastGroundingSnapshot
|
||||
|
||||
if (state.lastClickedCandidateId === candidateId) {
|
||||
return { isError: true, text: `Already clicked ${candidateId}` }
|
||||
}
|
||||
|
||||
const snapshotAge = Date.now() - new Date(snapshot.capturedAt).getTime()
|
||||
if (snapshotAge > 5000) {
|
||||
return { isError: true, text: `Stale snapshot (${Math.round(snapshotAge / 1000)}s)` }
|
||||
}
|
||||
|
||||
const snap = resolveSnapByCandidate(candidateId, snapshot)
|
||||
if (snap.source === 'none' && !snap.candidateId) {
|
||||
return { isError: true, text: `Not found: ${candidateId}` }
|
||||
}
|
||||
|
||||
const intent = {
|
||||
mode: 'execute' as const,
|
||||
candidateId,
|
||||
rawPoint: snap.rawPoint,
|
||||
snappedPoint: snap.snappedPoint,
|
||||
source: snap.source,
|
||||
confidence: snapshot.targetCandidates.find(c => c.id === candidateId)?.confidence ?? 0,
|
||||
path: [{ x: snap.snappedPoint.x, y: snap.snappedPoint.y, delayMs: 0 }],
|
||||
}
|
||||
stateManager.updatePointerIntent(intent, 't_test')
|
||||
stateManager.recordClickedCandidate(candidateId)
|
||||
|
||||
const candidate = snapshot.targetCandidates.find(c => c.id === candidateId)
|
||||
const bridgeConnected = browserDomBridge.getStatus().connected
|
||||
const routeDecision = candidate
|
||||
? decideBrowserAction(candidate, bridgeConnected)
|
||||
: { route: 'os_input' as const, reason: 'candidate not found' }
|
||||
|
||||
let executionRoute = routeDecision.route
|
||||
let routeNote = ''
|
||||
|
||||
if (routeDecision.route === 'browser_dom' && routeDecision.selector) {
|
||||
try {
|
||||
const frameIds = routeDecision.frameId !== undefined ? [routeDecision.frameId] : undefined
|
||||
if (routeDecision.bridgeMethod === 'checkCheckbox') {
|
||||
await browserDomBridge.checkCheckbox({ selector: routeDecision.selector, frameIds })
|
||||
}
|
||||
else {
|
||||
await browserDomBridge.clickSelector({ selector: routeDecision.selector, frameIds })
|
||||
}
|
||||
}
|
||||
catch (browserError) {
|
||||
executionRoute = 'os_input'
|
||||
routeNote = `browser-dom failed: ${browserError instanceof Error ? browserError.message : String(browserError)}`
|
||||
await executor.click({
|
||||
x: snap.snappedPoint.x,
|
||||
y: snap.snappedPoint.y,
|
||||
button: button || 'left',
|
||||
clickCount: clickCount ?? 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
else {
|
||||
await executor.click({
|
||||
x: snap.snappedPoint.x,
|
||||
y: snap.snappedPoint.y,
|
||||
button: button || 'left',
|
||||
clickCount: clickCount ?? 1,
|
||||
})
|
||||
}
|
||||
|
||||
const candidateDesc = candidate
|
||||
? `${candidate.source} ${candidate.role} "${candidate.label}"`
|
||||
: candidateId
|
||||
|
||||
const lines = [
|
||||
`Clicked: ${candidateDesc}`,
|
||||
` Snap: ${snap.reason}`,
|
||||
` Point: (${snap.snappedPoint.x}, ${snap.snappedPoint.y})`,
|
||||
` Route: ${executionRoute} (${routeDecision.reason})`,
|
||||
` Button: ${button || 'left'}, clicks: ${clickCount ?? 1}`,
|
||||
]
|
||||
if (routeNote)
|
||||
lines.push(` ⚠ ${routeNote}`)
|
||||
|
||||
return { isError: false, text: lines.join('\n'), executionRoute, routeNote }
|
||||
}
|
||||
|
||||
function freshSnapshot(candidates: DesktopTargetCandidate[]): DesktopGroundingSnapshot {
|
||||
return {
|
||||
snapshotId: 'dg_fresh',
|
||||
capturedAt: new Date().toISOString(), // fresh = now
|
||||
foregroundApp: 'Google Chrome',
|
||||
windows: [],
|
||||
screenshot: { dataBase64: '', mimeType: 'image/png', path: '', capturedAt: new Date().toISOString() },
|
||||
targetCandidates: candidates,
|
||||
staleFlags: { screenshot: false, ax: false, chromeSemantic: false },
|
||||
} as DesktopGroundingSnapshot
|
||||
}
|
||||
|
||||
function makeMockBridge(connected: boolean) {
|
||||
return {
|
||||
getStatus: () => ({ connected }),
|
||||
clickSelector: vi.fn().mockResolvedValue(undefined),
|
||||
checkCheckbox: vi.fn().mockResolvedValue(undefined),
|
||||
}
|
||||
}
|
||||
|
||||
function makeMockExecutor() {
|
||||
return {
|
||||
click: vi.fn().mockResolvedValue({}),
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// browser_dom routing: calls clickSelector
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('routes chrome_dom candidate through clickSelector when bridge is connected', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
selector: '#login-btn',
|
||||
frameId: 0,
|
||||
isPageContent: true,
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.executionRoute).toBe('browser_dom')
|
||||
expect(bridge.clickSelector).toHaveBeenCalledOnce()
|
||||
expect(bridge.clickSelector).toHaveBeenCalledWith({
|
||||
selector: '#login-btn',
|
||||
frameIds: [0],
|
||||
})
|
||||
expect(executor.click).not.toHaveBeenCalled()
|
||||
expect(result.text).toContain('Route: browser_dom')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// browser_dom fallback: clickSelector fails → executor.click
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('falls back to OS click when clickSelector throws', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
selector: '#broken',
|
||||
frameId: 0,
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
bridge.clickSelector.mockRejectedValue(new Error('Element not found'))
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.executionRoute).toBe('os_input')
|
||||
expect(bridge.clickSelector).toHaveBeenCalledOnce()
|
||||
expect(executor.click).toHaveBeenCalledOnce()
|
||||
expect(result.text).toContain('Route: os_input')
|
||||
expect(result.text).toContain('browser-dom failed')
|
||||
expect(result.text).toContain('Element not found')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// checkbox: routes to checkCheckbox, not clickSelector
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('dispatches to checkCheckbox for checkbox candidates', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
tag: 'input',
|
||||
inputType: 'checkbox',
|
||||
role: 'checkbox',
|
||||
selector: '#agree',
|
||||
frameId: 0,
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
expect(bridge.checkCheckbox).toHaveBeenCalledOnce()
|
||||
expect(bridge.checkCheckbox).toHaveBeenCalledWith({
|
||||
selector: '#agree',
|
||||
frameIds: [0],
|
||||
})
|
||||
expect(bridge.clickSelector).not.toHaveBeenCalled()
|
||||
expect(executor.click).not.toHaveBeenCalled()
|
||||
expect(result.text).toContain('Route: browser_dom')
|
||||
expect(result.text).toContain('checkCheckbox')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// AX candidate: bypasses browser-dom entirely
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('routes AX candidate directly to OS click, never touches bridge', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'ax',
|
||||
role: 'AXButton',
|
||||
label: 'Close',
|
||||
selector: undefined,
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.executionRoute).toBe('os_input')
|
||||
expect(bridge.clickSelector).not.toHaveBeenCalled()
|
||||
expect(bridge.checkCheckbox).not.toHaveBeenCalled()
|
||||
expect(executor.click).toHaveBeenCalledOnce()
|
||||
expect(result.text).toContain('Route: os_input')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Bridge disconnected: chrome_dom candidate falls back to OS
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('routes chrome_dom to OS click when bridge is disconnected', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
selector: '#btn',
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(false) // disconnected
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.executionRoute).toBe('os_input')
|
||||
expect(bridge.clickSelector).not.toHaveBeenCalled()
|
||||
expect(executor.click).toHaveBeenCalledOnce()
|
||||
expect(result.text).toContain('not connected')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// No selector: chrome_dom candidate without selector → OS click
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('routes chrome_dom without selector to OS click', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
selector: undefined,
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.executionRoute).toBe('os_input')
|
||||
expect(bridge.clickSelector).not.toHaveBeenCalled()
|
||||
expect(executor.click).toHaveBeenCalledOnce()
|
||||
expect(result.text).toContain('no CSS selector')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Duplicate click guard
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('blocks duplicate click on same candidate without re-observe', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({ id: 't_0' })
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
// First click succeeds
|
||||
const first = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
expect(first.isError).toBe(false)
|
||||
|
||||
// Second click on same candidate without re-observe → blocked
|
||||
const second = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
expect(second.isError).toBe(true)
|
||||
expect(second.text).toContain('Already clicked')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Duplicate guard reset after re-observe
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('allows same candidate click after re-observe', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({ id: 't_0' })
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
// First click
|
||||
await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
// Re-observe resets the guard
|
||||
sm.updateGroundingSnapshot(freshSnapshot([makeCandidate({ id: 't_0' })]))
|
||||
|
||||
// Click again after re-observe → allowed
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
expect(result.isError).toBe(false)
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Stale snapshot rejection
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('rejects click on stale snapshot (>5s)', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({ id: 't_0' })
|
||||
const staleSnapshot = {
|
||||
...freshSnapshot([candidate]),
|
||||
capturedAt: new Date(Date.now() - 10_000).toISOString(), // 10s ago
|
||||
}
|
||||
sm.updateGroundingSnapshot(staleSnapshot)
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.text).toContain('Stale')
|
||||
expect(bridge.clickSelector).not.toHaveBeenCalled()
|
||||
expect(executor.click).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Missing candidate
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('returns error for non-existent candidate id', async () => {
|
||||
const sm = new RunStateManager()
|
||||
sm.updateGroundingSnapshot(freshSnapshot([makeCandidate({ id: 't_0' })]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_99',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.text).toContain('Not found')
|
||||
expect(bridge.clickSelector).not.toHaveBeenCalled()
|
||||
expect(executor.click).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// No snapshot
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('returns error when no snapshot exists', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.text).toContain('No snapshot')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Frame ID passthrough
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('passes non-zero frameId to clickSelector', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
selector: '#iframe-btn',
|
||||
frameId: 5,
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(bridge.clickSelector).toHaveBeenCalledWith({
|
||||
selector: '#iframe-btn',
|
||||
frameIds: [5],
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// checkCheckbox fallback on failure
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('falls back to OS click when checkCheckbox throws', async () => {
|
||||
const sm = new RunStateManager()
|
||||
const candidate = makeCandidate({
|
||||
id: 't_0',
|
||||
source: 'chrome_dom',
|
||||
tag: 'input',
|
||||
inputType: 'checkbox',
|
||||
role: 'checkbox',
|
||||
selector: '#cb',
|
||||
frameId: 0,
|
||||
})
|
||||
sm.updateGroundingSnapshot(freshSnapshot([candidate]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
bridge.checkCheckbox.mockRejectedValue(new Error('checkbox toggle failed'))
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const result = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.executionRoute).toBe('os_input')
|
||||
expect(bridge.checkCheckbox).toHaveBeenCalledOnce()
|
||||
expect(executor.click).toHaveBeenCalledOnce()
|
||||
expect(result.text).toContain('browser-dom failed')
|
||||
expect(result.text).toContain('checkbox toggle failed')
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Two candidates: click different ones in sequence
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
it('allows clicking different candidates in sequence', async () => {
|
||||
const sm = new RunStateManager()
|
||||
sm.updateGroundingSnapshot(freshSnapshot([
|
||||
makeCandidate({ id: 't_0', selector: '#first', label: 'First' }),
|
||||
makeCandidate({ id: 't_1', selector: '#second', label: 'Second' }),
|
||||
]))
|
||||
|
||||
const bridge = makeMockBridge(true)
|
||||
const executor = makeMockExecutor()
|
||||
|
||||
const first = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_0',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
expect(first.isError).toBe(false)
|
||||
expect(first.text).toContain('First')
|
||||
|
||||
const second = await simulateClickTargetHandler({
|
||||
stateManager: sm,
|
||||
candidateId: 't_1',
|
||||
browserDomBridge: bridge,
|
||||
executor,
|
||||
})
|
||||
expect(second.isError).toBe(false)
|
||||
expect(second.text).toContain('Second')
|
||||
expect(bridge.clickSelector).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
|
||||
|
||||
import type { PointerIntent } from '../desktop-grounding-types'
|
||||
import type { ExecuteAction } from './action-executor'
|
||||
import type { ComputerUseServerRuntime } from './runtime'
|
||||
|
||||
@@ -21,7 +22,9 @@ import process from 'node:process'
|
||||
|
||||
import { z } from 'zod'
|
||||
|
||||
import { decideBrowserAction } from '../browser-action-router'
|
||||
import { captureDesktopGrounding, formatGroundingForAgent } from '../desktop-grounding'
|
||||
import { resolveSnapByCandidate } from '../snap-resolver'
|
||||
import { textContent } from './content'
|
||||
import { registerToolWithDescriptor, requireDescriptor } from './tool-descriptors/register-helper'
|
||||
|
||||
@@ -74,15 +77,7 @@ export function registerDesktopGroundingTools(params: {
|
||||
})
|
||||
|
||||
// Update RunState — grounding snapshot
|
||||
runtime.stateManager.updateGroundingSnapshot({
|
||||
...snapshot,
|
||||
screenshot: snapshot.screenshot
|
||||
? {
|
||||
...snapshot.screenshot,
|
||||
dataBase64: '',
|
||||
}
|
||||
: snapshot.screenshot,
|
||||
})
|
||||
runtime.stateManager.updateGroundingSnapshot(snapshot)
|
||||
|
||||
// Also update screenshot state so desktop_get_state and other
|
||||
// tools can see the latest screenshot from this observation
|
||||
@@ -123,7 +118,6 @@ export function registerDesktopGroundingTools(params: {
|
||||
return { content }
|
||||
}
|
||||
catch (error) {
|
||||
runtime.stateManager.clearGroundingState()
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return {
|
||||
content: [textContent(`desktop_observe failed: ${message}`)],
|
||||
@@ -147,14 +141,181 @@ export function registerDesktopGroundingTools(params: {
|
||||
},
|
||||
|
||||
handler: async ({ candidateId, clickCount, button }) => {
|
||||
return await executeAction({
|
||||
kind: 'desktop_click_target',
|
||||
input: {
|
||||
try {
|
||||
const state = runtime.stateManager.getState()
|
||||
|
||||
// Validate: must have a recent grounding snapshot
|
||||
if (!state.lastGroundingSnapshot) {
|
||||
return {
|
||||
content: [textContent('ERROR: No desktop_observe snapshot available. Call desktop_observe first to get a list of target candidates.')],
|
||||
isError: true,
|
||||
}
|
||||
}
|
||||
|
||||
const snapshot = state.lastGroundingSnapshot
|
||||
|
||||
// Validate: check for duplicate clicks on same candidate without re-observe
|
||||
if (state.lastClickedCandidateId === candidateId) {
|
||||
return {
|
||||
content: [textContent(`WARNING: You already clicked candidate "${candidateId}" without calling desktop_observe again. Call desktop_observe to refresh the state before clicking the same target.`)],
|
||||
isError: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Validate: check snapshot staleness (>5s)
|
||||
const snapshotAge = Date.now() - new Date(snapshot.capturedAt).getTime()
|
||||
if (snapshotAge > 5000) {
|
||||
return {
|
||||
content: [textContent(`WARNING: Grounding snapshot "${snapshot.snapshotId}" is ${Math.round(snapshotAge / 1000)}s old. Call desktop_observe to get a fresh snapshot before clicking.`)],
|
||||
isError: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve snap
|
||||
const snap = resolveSnapByCandidate(candidateId, snapshot)
|
||||
|
||||
if (snap.source === 'none' && !snap.candidateId) {
|
||||
return {
|
||||
content: [textContent(`ERROR: Candidate "${candidateId}" not found in snapshot "${snapshot.snapshotId}". Available candidates: ${snapshot.targetCandidates.map(c => c.id).join(', ')}`)],
|
||||
isError: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Build pointer intent
|
||||
const intent: PointerIntent = {
|
||||
mode: 'execute',
|
||||
candidateId,
|
||||
clickCount,
|
||||
button,
|
||||
},
|
||||
}, 'desktop_click_target')
|
||||
rawPoint: snap.rawPoint,
|
||||
snappedPoint: snap.snappedPoint,
|
||||
source: snap.source,
|
||||
confidence: snapshot.targetCandidates.find(c => c.id === candidateId)?.confidence ?? 0,
|
||||
path: [
|
||||
{ x: snap.snappedPoint.x, y: snap.snappedPoint.y, delayMs: 0 },
|
||||
],
|
||||
}
|
||||
|
||||
// Update RunState — pointer intent
|
||||
runtime.stateManager.updatePointerIntent(intent, candidateId)
|
||||
|
||||
// Route the click: browser-dom for chrome_dom candidates, OS input for everything else.
|
||||
// Pass button and clickCount so non-left or multi-click requests fall through to OS input
|
||||
// rather than silently degrading to a single left click on the browser-dom path.
|
||||
const candidate = snapshot.targetCandidates.find(c => c.id === candidateId)
|
||||
const bridgeConnected = runtime.browserDomBridge?.getStatus().connected ?? false
|
||||
const routeDecision = candidate
|
||||
? decideBrowserAction(candidate, bridgeConnected, button, clickCount)
|
||||
: { route: 'os_input' as const, reason: 'candidate not found' }
|
||||
|
||||
let executionRoute = routeDecision.route
|
||||
let routeNote = ''
|
||||
|
||||
if (routeDecision.route === 'browser_dom' && routeDecision.selector) {
|
||||
// Try browser-dom bridge action first, dispatching by method
|
||||
try {
|
||||
const frameIds = routeDecision.frameId !== undefined ? [routeDecision.frameId] : undefined
|
||||
if (routeDecision.bridgeMethod === 'checkCheckbox') {
|
||||
const frameResults = await runtime.browserDomBridge!.checkCheckbox({
|
||||
selector: routeDecision.selector,
|
||||
frameIds,
|
||||
})
|
||||
// NOTICE: bridge resolve ≠ DOM success. Each frame returns
|
||||
// { success, error } — if none succeeded the selector/frame was
|
||||
// stale and we must fall back to OS click.
|
||||
const anySucceeded = Array.isArray(frameResults) && frameResults.some(
|
||||
fr => (fr.result as Record<string, unknown>)?.success === true,
|
||||
)
|
||||
if (!anySucceeded) {
|
||||
throw new Error('checkCheckbox: no frame reported success')
|
||||
}
|
||||
}
|
||||
else {
|
||||
const clickResult = await runtime.browserDomBridge!.clickSelector({
|
||||
selector: routeDecision.selector,
|
||||
frameIds,
|
||||
})
|
||||
// NOTICE: clickSelector resolves even when clickAt hits no element.
|
||||
// Check per-frame results; if none succeeded, fall back to OS click.
|
||||
const clickFrames = clickResult?.clickResults
|
||||
const anyClickSucceeded = Array.isArray(clickFrames) && clickFrames.some(
|
||||
fr => (fr.result as Record<string, unknown>)?.success === true,
|
||||
)
|
||||
if (!anyClickSucceeded) {
|
||||
throw new Error('clickSelector: no frame reported a successful click')
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (browserError) {
|
||||
// Fallback to OS input on browser-dom failure; still goes through policy pipeline
|
||||
executionRoute = 'os_input'
|
||||
routeNote = `browser-dom ${routeDecision.bridgeMethod ?? 'click'} failed (${browserError instanceof Error ? browserError.message : String(browserError)}), fell back to OS input`
|
||||
const actionResult = await executeAction({
|
||||
kind: 'click',
|
||||
input: {
|
||||
x: snap.snappedPoint.x,
|
||||
y: snap.snappedPoint.y,
|
||||
button: button || 'left',
|
||||
clickCount: clickCount ?? 1,
|
||||
},
|
||||
}, 'desktop_click_target')
|
||||
// If the action was denied or queued for approval, relay the policy result
|
||||
// and do not report a false success or update post-click state.
|
||||
const status = (actionResult.structuredContent as Record<string, unknown> | undefined)?.status
|
||||
if (actionResult.isError || status === 'approval_required' || status === 'denied') {
|
||||
return actionResult
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// OS-level click through policy pipeline — respects approvalMode and policy gates
|
||||
const actionResult = await executeAction({
|
||||
kind: 'click',
|
||||
input: {
|
||||
x: snap.snappedPoint.x,
|
||||
y: snap.snappedPoint.y,
|
||||
button: button || 'left',
|
||||
clickCount: clickCount ?? 1,
|
||||
},
|
||||
}, 'desktop_click_target')
|
||||
// If the action was denied or queued for approval, relay the policy result
|
||||
// and do not report a false success or update post-click state.
|
||||
const status = (actionResult.structuredContent as Record<string, unknown> | undefined)?.status
|
||||
if (actionResult.isError || status === 'approval_required' || status === 'denied') {
|
||||
return actionResult
|
||||
}
|
||||
}
|
||||
|
||||
// Record the candidate as clicked only after execution succeeds or bypasses policy
|
||||
runtime.stateManager.recordClickedCandidate(candidateId)
|
||||
|
||||
const candidateDesc = candidate ? `${candidate.source} ${candidate.role} "${candidate.label}"` : candidateId
|
||||
|
||||
const lines = [
|
||||
`Clicked: ${candidateDesc}`,
|
||||
` Snap: ${snap.reason}`,
|
||||
` Point: (${snap.snappedPoint.x}, ${snap.snappedPoint.y})`,
|
||||
` Route: ${executionRoute} (${routeDecision.reason})`,
|
||||
` Button: ${button || 'left'}, clicks: ${clickCount ?? 1}`,
|
||||
]
|
||||
|
||||
if (routeNote) {
|
||||
lines.push(` ⚠ ${routeNote}`)
|
||||
}
|
||||
|
||||
if (snap.reason.includes('stale')) {
|
||||
lines.push(' ⚠ WARNING: Target source is stale. Consider calling desktop_observe again.')
|
||||
}
|
||||
|
||||
return {
|
||||
content: [textContent(lines.join('\n'))],
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return {
|
||||
content: [textContent(`desktop_click_target failed: ${message}`)],
|
||||
isError: true,
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -561,6 +561,29 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
|
||||
tabId,
|
||||
frameIds,
|
||||
})
|
||||
|
||||
// NOTICE: clickSelector resolves even when the clickAt step misses
|
||||
// (e.g. reflow between target lookup and click dispatch). Inspect
|
||||
// per-frame results before reporting success.
|
||||
const clickFrames = result?.clickResults
|
||||
const anyClickSucceeded = Array.isArray(clickFrames) && clickFrames.some(
|
||||
fr => (fr.result as Record<string, unknown>)?.success === true,
|
||||
)
|
||||
if (!anyClickSucceeded) {
|
||||
return {
|
||||
isError: true,
|
||||
content: [
|
||||
textContent(`browser_dom_click: clicked at (${result.targetPoint.x}, ${result.targetPoint.y}) in frame ${result.targetFrameId} but no frame reported a successful DOM click for "${selector}".`),
|
||||
],
|
||||
structuredContent: {
|
||||
status: 'click_miss',
|
||||
selector,
|
||||
...result,
|
||||
bridge: runtime.browserDomBridge.getStatus(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
content: [
|
||||
textContent(`Clicked selector "${selector}" in frame ${result.targetFrameId} at (${result.targetPoint.x}, ${result.targetPoint.y}).`),
|
||||
|
||||
@@ -36,7 +36,8 @@ export type ToolKind
|
||||
|
||||
/**
|
||||
* Tool descriptor defines the canonical metadata for a single MCP tool.
|
||||
* All fields are required (fail-closed policy). No field may be left undefined.
|
||||
* All fields except `defaultDeferred` are required (fail-closed policy).
|
||||
* `defaultDeferred` defaults to false when omitted.
|
||||
*/
|
||||
export interface ToolDescriptor {
|
||||
/**
|
||||
|
||||
@@ -417,6 +417,14 @@ export class RunStateManager {
|
||||
this.touch()
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the candidate id that was just successfully clicked.
|
||||
*/
|
||||
recordClickedCandidate(candidateId: string): void {
|
||||
this.state.lastClickedCandidateId = candidateId
|
||||
this.touch()
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear desktop grounding state when the snapshot becomes invalid.
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user