diff --git a/services/computer-use-mcp/chrome-extension/background.js b/services/computer-use-mcp/chrome-extension/background.js index d8a154a62..a3b021ea7 100644 --- a/services/computer-use-mcp/chrome-extension/background.js +++ b/services/computer-use-mcp/chrome-extension/background.js @@ -28,6 +28,8 @@ const AIRI_BRIDGE_HELLO = { } const BRIDGE_RECONNECT_MIN_MS = 1_000 const BRIDGE_RECONNECT_MAX_MS = 10_000 +const SEND_CU_ACTION_TIMEOUT_MS = 8_000 +const WAIT_FOR_ELEMENT_POLL_INTERVAL_MS = 500 let bridgeSocket = null let bridgeReconnectDelayMs = BRIDGE_RECONNECT_MIN_MS @@ -182,7 +184,7 @@ async function sendCUAction(tabId, frameId, method, args) { return new Promise((resolve) => { const timeout = setTimeout(() => { resolve({ success: false, error: 'sendMessage timeout' }) - }, 8000) + }, SEND_CU_ACTION_TIMEOUT_MS) try { chrome.tabs.sendMessage( @@ -400,6 +402,9 @@ async function readAllFramesDOMWithOffsets(tabId, frameIds, opts) { * - findElements: find multiple elements by CSS selector * - getClickTarget: get center point of an element for click targeting * - getElementAttributes: get all attributes of an element + * - readInputValue: read the current value of an input/textarea/select + * - getComputedStyles: read computed CSS styles of an element + * - waitForElement: poll until a CSS selector matches in any frame */ async function handleCommand(cmd) { const { action, id } = cmd @@ -441,6 +446,91 @@ async function handleCommand(cmd) { result = await runCUAction(tabId, cmd.frameIds || null, 'getElementAttributes', [cmd.selector || '']) break + case 'readInputValue': + result = await runCUAction(tabId, cmd.frameIds || null, 'readInputValue', [cmd.selector || '']) + break + + case 'getComputedStyles': + result = await runCUAction(tabId, cmd.frameIds || null, 'getComputedStyles', [cmd.selector || '', cmd.properties || null]) + break + + case 'waitForElement': { + // Background-level polling: repeatedly call findElements until a match + // is found or the timeout expires. No DOM MutationObserver needed. + const selector = cmd.selector || '' + const timeoutMs = Math.min(Math.max(Number(cmd.timeoutMs) || 10_000, 500), 30_000) + const deadline = Date.now() + timeoutMs + let lastFrames = [] + let lastPollError = '' + let lastFrameError = '' + + result = await new Promise((resolve) => { + async function poll() { + try { + const frames = await runCUAction(tabId, cmd.frameIds || null, 'findElements', [selector, 1]) + lastFrames = frames + const frameErrors = frames + .map(entry => unwrapBridgePayload(entry.result)) + .filter(payload => payload && payload.success === false && typeof payload.error === 'string') + .map(payload => payload.error) + if (frameErrors.length > 0) { + lastFrameError = frameErrors.join('; ') + } + + const found = frames.some((entry) => { + const payload = unwrapBridgePayload(entry.result) + return payload && payload.success && Array.isArray(payload.elements) && payload.elements.length > 0 + }) + if (found) { + resolve(frames) + return + } + } + catch (e) { + lastPollError = e?.message || String(e) + } + + if (Date.now() >= deadline) { + if (lastFrames.length === 0) { + let frameIds = [] + if (Array.isArray(cmd.frameIds) && cmd.frameIds.length > 0) { + frameIds = cmd.frameIds + } + else if (typeof cmd.frameIds === 'number') { + frameIds = [cmd.frameIds] + } + else { + try { + const frames = await chrome.webNavigation.getAllFrames({ tabId }) + frameIds = frames.map(frame => frame.frameId) + } + catch { + frameIds = [0] + } + } + lastFrames = frameIds.map(frameId => ({ frameId })) + } + + const lastError = lastPollError || lastFrameError || undefined + resolve(lastFrames.map(entry => ({ + frameId: entry.frameId, + result: { + success: false, + error: `timed out waiting for selector "${selector}"`, + selector, + timeoutMs, + ...(lastError ? { lastError } : {}), + }, + }))) + return + } + setTimeout(poll, WAIT_FOR_ELEMENT_POLL_INTERVAL_MS) + } + poll() + }) + break + } + default: return { id, ok: false, error: `unknown action: ${action}` } } diff --git a/services/computer-use-mcp/chrome-extension/content.js b/services/computer-use-mcp/chrome-extension/content.js index 11d9a991d..ba360bbfe 100644 --- a/services/computer-use-mcp/chrome-extension/content.js +++ b/services/computer-use-mcp/chrome-extension/content.js @@ -243,6 +243,98 @@ return { success: false, error: e.message } } }, + + /** + * Read the current value of an input, textarea, or select element. + * Returns value plus basic element metadata. Read-only: no DOM mutation. + */ + readInputValue(selector) { + try { + const el = document.querySelector(selector) + if (!el) + return { success: false, error: 'not found' } + + const tag = el.tagName.toLowerCase() + if (tag !== 'input' && tag !== 'textarea' && tag !== 'select') + return { success: false, error: 'element is not an input, textarea, or select' } + + const type = typeof el.type === 'string' ? el.type : '' + const rawValue = String(el.value ?? '') + const isPassword = tag === 'input' && type.toLowerCase() === 'password' + const result = { + value: isPassword ? '[redacted]' : rawValue.slice(0, 60), + valueLength: rawValue.length, + valueRedacted: isPassword, + valueTruncated: !isPassword && rawValue.length > 60, + tag, + id: el.id || '', + name: el.name || '', + type, + } + + if (tag === 'input' && (type === 'checkbox' || type === 'radio')) { + result.checked = !!el.checked + } + + if (tag === 'select') { + result.selectedIndex = el.selectedIndex + result.selectedText = el.selectedIndex >= 0 && el.options[el.selectedIndex] + ? el.options[el.selectedIndex].text.slice(0, 120) + : '' + } + + return { success: true, ...result } + } + catch (e) { + return { success: false, error: e.message } + } + }, + + /** + * Read computed CSS styles of an element. + * If `properties` is provided, return only those properties. + * Otherwise return a controlled default set to avoid dumping the full + * CSSStyleDeclaration (which can be multi-KB). + */ + getComputedStyles(selector, properties) { + const DEFAULT_PROPERTIES = [ + 'display', + 'visibility', + 'opacity', + 'position', + 'width', + 'height', + 'color', + 'background-color', + 'font-size', + 'font-family', + 'overflow', + 'z-index', + 'pointer-events', + 'cursor', + ] + + try { + const el = document.querySelector(selector) + if (!el) + return { success: false, error: 'not found' } + + const computed = window.getComputedStyle(el) + const props = Array.isArray(properties) && properties.length > 0 + ? properties + : DEFAULT_PROPERTIES + const styles = Object.create(null) + + for (const prop of props) { + styles[prop] = computed.getPropertyValue(prop) + } + + return { success: true, styles } + } + catch (e) { + return { success: false, error: e.message } + } + }, } window.__AIRI_DG__ = __AIRI_DG__ diff --git a/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts b/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts index ddc6ec1a6..20b7b0950 100644 --- a/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts +++ b/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts @@ -3,6 +3,73 @@ import { WebSocket, WebSocketServer } from 'ws' import { BrowserDomExtensionBridge } from './extension-bridge' +/** + * Helper: create a bridge + client pair, wait for the client to connect + * and send the hello handshake. + */ +async function createConnectedBridge(config?: Partial<{ + requestTimeoutMs: number +}>) { + const bridge = new BrowserDomExtensionBridge({ + enabled: true, + host: '127.0.0.1', + port: 0, + requestTimeoutMs: config?.requestTimeoutMs ?? 1_000, + }) + await bridge.start() + + const status = bridge.getStatus() + const client = new WebSocket(`ws://${status.host}:${status.port}`) + + await new Promise((resolve, reject) => { + client.once('open', () => { + client.send(JSON.stringify({ + type: 'hello', + source: 'test-extension', + version: 'bridge-test', + })) + resolve() + }) + client.once('error', reject) + }) + + return { bridge, client } +} + +/** + * Helper: register a mock handler on the client that echoes a fixed + * result for a given action name. + */ +function mockClientAction( + client: WebSocket, + actionName: string, + resultFn: (data: Record) => unknown, + opts?: { delayMs?: number }, +) { + client.on('message', (raw) => { + const data = JSON.parse(String(raw)) as Record + if (typeof data.id !== 'string') + return + if (data.action !== actionName) + return + + const respond = () => { + client.send(JSON.stringify({ + id: data.id, + ok: true, + result: resultFn(data), + })) + } + + if (opts?.delayMs) { + setTimeout(respond, opts.delayMs) + } + else { + respond() + } + }) +} + describe('browserDomExtensionBridge', () => { let bridge: BrowserDomExtensionBridge | undefined let client: WebSocket | undefined @@ -22,45 +89,14 @@ describe('browserDomExtensionBridge', () => { }) it('round-trips actions over the extension websocket bridge', async () => { - bridge = new BrowserDomExtensionBridge({ - enabled: true, - host: '127.0.0.1', - port: 0, - requestTimeoutMs: 1_000, - }) - await bridge.start() + const result = await createConnectedBridge() + bridge = result.bridge + client = result.client - const status = bridge.getStatus() - client = new WebSocket(`ws://${status.host}:${status.port}`) - - client.on('message', (raw) => { - const data = JSON.parse(String(raw)) as Record - if (typeof data.id !== 'string') - return - - if (data.action === 'getActiveTab') { - client!.send(JSON.stringify({ - id: data.id, - ok: true, - result: { - title: 'AIRI Demo Tab', - url: 'https://example.com/demo', - }, - })) - } - }) - - await new Promise((resolve, reject) => { - client!.once('open', () => { - client!.send(JSON.stringify({ - type: 'hello', - source: 'test-extension', - version: 'bridge-test', - })) - resolve() - }) - client!.once('error', reject) - }) + mockClientAction(client, 'getActiveTab', () => ({ + title: 'AIRI Demo Tab', + url: 'https://example.com/demo', + })) const activeTab = await bridge.getActiveTab() @@ -72,98 +108,183 @@ describe('browserDomExtensionBridge', () => { expect(bridge.getStatus().lastHello?.source).toBe('test-extension') }) - it('rejects clickSelector on the read-only extension transport even when getClickTarget succeeds', async () => { + it('supportsAction returns true for read-only actions and false for mutating actions', () => { bridge = new BrowserDomExtensionBridge({ enabled: true, host: '127.0.0.1', port: 0, requestTimeoutMs: 1_000, }) - await bridge.start() - const status = bridge.getStatus() - client = new WebSocket(`ws://${status.host}:${status.port}`) + // Read-only actions should be supported + expect(bridge.supportsAction('readInputValue')).toBe(true) + expect(bridge.supportsAction('getComputedStyles')).toBe(true) + expect(bridge.supportsAction('waitForElement')).toBe(true) + expect(bridge.supportsAction('getActiveTab')).toBe(true) + expect(bridge.supportsAction('findElements')).toBe(true) + expect(bridge.supportsAction('getElementAttributes')).toBe(true) - client.on('message', (raw) => { - const data = JSON.parse(String(raw)) as Record - if (typeof data.id !== 'string') - return - - if (data.action === 'getClickTarget') { - client!.send(JSON.stringify({ - id: data.id, - ok: true, - result: [ - { - frameId: 5, - result: { - success: true, - x: 321, - y: 182, - element: { - tag: 'button', - text: 'Submit', - }, - center: { - x: 321, - y: 182, - }, - }, - }, - ], - })) - return - } - - if (data.action === 'clickAt') { - client!.send(JSON.stringify({ - id: data.id, - ok: true, - result: [ - { - frameId: 5, - result: { - success: true, - }, - }, - ], - })) - } - }) - - await new Promise((resolve, reject) => { - client!.once('open', () => { - client!.send(JSON.stringify({ - type: 'hello', - source: 'test-extension', - version: 'bridge-test', - })) - resolve() - }) - client!.once('error', reject) - }) - - await expect(bridge.clickSelector({ - selector: '#submit', - frameIds: [5], - })).rejects.toThrow('does not support action "clickAt"') + // Mutating actions should NOT be supported + expect(bridge.supportsAction('setInputValue')).toBe(false) + expect(bridge.supportsAction('checkCheckbox')).toBe(false) + expect(bridge.supportsAction('selectOption')).toBe(false) + expect(bridge.supportsAction('triggerEvent')).toBe(false) + expect(bridge.supportsAction('clickAt')).toBe(false) }) - it('rejects unsupported DOM-mutating actions before sending them to the extension transport', async () => { - bridge = new BrowserDomExtensionBridge({ - enabled: true, - host: '127.0.0.1', - port: 0, - requestTimeoutMs: 1_000, - }) - await bridge.start() + it('readInputValue round-trips through the bridge', async () => { + const result = await createConnectedBridge() + bridge = result.bridge + client = result.client - expect(bridge.supportsAction('readAllFramesDOM')).toBe(true) - expect(bridge.supportsAction('setInputValue')).toBe(false) - await expect(bridge.setInputValue({ - selector: '#email', - value: 'hello@example.com', - })).rejects.toThrow('does not support action "setInputValue"') + mockClientAction(client, 'readInputValue', () => ([ + { + frameId: 0, + result: { + success: true, + value: 'hello world', + tag: 'input', + id: 'search', + name: 'q', + type: 'text', + }, + }, + ])) + + const frames = await bridge.readInputValue({ selector: '#search' }) + + expect(frames).toEqual([ + { + frameId: 0, + result: { + success: true, + value: 'hello world', + tag: 'input', + id: 'search', + name: 'q', + type: 'text', + }, + }, + ]) + }) + + it('getComputedStyles round-trips through the bridge', async () => { + const result = await createConnectedBridge() + bridge = result.bridge + client = result.client + + mockClientAction(client, 'getComputedStyles', () => ([ + { + frameId: 0, + result: { + success: true, + styles: { + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + ])) + + const frames = await bridge.getComputedStyles({ + selector: '.container', + properties: ['display', 'visibility', 'opacity'], + }) + + expect(frames).toEqual([ + { + frameId: 0, + result: { + success: true, + styles: { + display: 'block', + visibility: 'visible', + opacity: '1', + }, + }, + }, + ]) + }) + + it('waitForElement uses action-specific timeout, not default requestTimeoutMs', async () => { + // Configure bridge with a very short default timeout (500ms) + const result = await createConnectedBridge({ requestTimeoutMs: 500 }) + bridge = result.bridge + client = result.client + + // Mock: respond after 800ms — longer than the 500ms default but well + // within the 3000ms action-specific timeout we'll pass. + mockClientAction(client, 'waitForElement', () => ([ + { + frameId: 0, + result: { success: true, elements: [{ tag: 'div', id: 'lazy' }] }, + }, + ]), { delayMs: 800 }) + + // This should NOT reject at 500ms because waitForElement uses a + // bridge-level timeout override that covers extension-side polling. + const frames = await bridge.waitForElement({ + selector: '#lazy', + timeoutMs: 3_000, + }) + + expect(frames).toEqual([ + { + frameId: 0, + result: { success: true, elements: [{ tag: 'div', id: 'lazy' }] }, + }, + ]) + }) + + it('waitForElement uses the default requestTimeoutMs for extension-side polling when no timeoutMs is provided', async () => { + const result = await createConnectedBridge({ requestTimeoutMs: 200 }) + bridge = result.bridge + client = result.client + + mockClientAction(client, 'waitForElement', data => ([ + { + frameId: 0, + result: { + success: false, + selector: data.selector, + timeoutMs: data.timeoutMs, + error: 'timed out waiting for selector "#missing"', + }, + }, + ]), { delayMs: 300 }) + + const frames = await bridge.waitForElement({ selector: '#missing' }) + + expect(frames).toEqual([ + { + frameId: 0, + result: { + success: false, + selector: '#missing', + timeoutMs: 200, + error: 'timed out waiting for selector "#missing"', + }, + }, + ]) + }) + + it('rejects pending requests when the bridge disconnects', async () => { + const result = await createConnectedBridge() + bridge = result.bridge + client = result.client + + // Start a request but don't respond to it + const promise = bridge.getActiveTab() + expect(bridge.getStatus().pendingRequests).toBe(1) + + // Close the client to simulate disconnection + client.close() + client = undefined + + await expect(promise).rejects.toThrow(/disconnected/) + expect(bridge.getStatus().connected).toBe(false) + expect(bridge.getStatus().pendingRequests).toBe(0) }) it('can retry startup after an initial bind failure', async () => { @@ -213,43 +334,4 @@ describe('browserDomExtensionBridge', () => { await new Promise(resolve => setTimeout(resolve, 10)) expect(bridge.getStatus().connected).toBe(true) }) - - it('rejects in-flight requests immediately when the socket disconnects', async () => { - bridge = new BrowserDomExtensionBridge({ - enabled: true, - host: '127.0.0.1', - port: 0, - requestTimeoutMs: 10_000, - }) - await bridge.start() - - const status = bridge.getStatus() - client = new WebSocket(`ws://${status.host}:${status.port}`) - - client.on('message', (raw) => { - const data = JSON.parse(String(raw)) as Record - if (data.action === 'getActiveTab') { - client!.close() - } - }) - - await new Promise((resolve, reject) => { - client!.once('open', () => { - client!.send(JSON.stringify({ - type: 'hello', - source: 'test-extension', - version: 'bridge-test', - })) - resolve() - }) - client!.once('error', reject) - }) - - const startedAt = Date.now() - - await expect(bridge.getActiveTab()).rejects.toThrow('browser dom bridge disconnected before completing pending request') - expect(Date.now() - startedAt).toBeLessThan(1_000) - expect(bridge.getStatus().pendingRequests).toBe(0) - expect(bridge.getStatus().connected).toBe(false) - }) }) diff --git a/services/computer-use-mcp/src/browser-dom/extension-bridge.ts b/services/computer-use-mcp/src/browser-dom/extension-bridge.ts index 5f6fa5621..71abdf34f 100644 --- a/services/computer-use-mcp/src/browser-dom/extension-bridge.ts +++ b/services/computer-use-mcp/src/browser-dom/extension-bridge.ts @@ -19,7 +19,11 @@ const SUPPORTED_ACTIONS = new Set([ 'findElements', 'getClickTarget', 'getElementAttributes', + 'readInputValue', + 'getComputedStyles', + 'waitForElement', ]) +const WAIT_FOR_ELEMENT_BRIDGE_TIMEOUT_BUFFER_MS = 9_500 interface PendingBridgeRequest { reject: (error: Error) => void @@ -176,7 +180,11 @@ export class BrowserDomExtensionBridge { } } - async callAction(action: string, payload: Record = {}): Promise { + async callAction( + action: string, + payload: Record = {}, + options?: { timeoutMs?: number }, + ): Promise { if (!this.config.enabled) { throw new Error('browser dom bridge is disabled') } @@ -196,12 +204,14 @@ export class BrowserDomExtensionBridge { ...payload, } + const effectiveTimeoutMs = options?.timeoutMs ?? this.config.requestTimeoutMs + const result = await new Promise((resolve, reject) => { const timeoutId = setTimeout(() => { this.pending.delete(id) this.status.pendingRequests = this.pending.size reject(new Error(`browser dom bridge timed out waiting for ${action}`)) - }, this.config.requestTimeoutMs) + }, effectiveTimeoutMs) this.pending.set(id, { resolve: value => resolve(value as TResult), @@ -364,11 +374,18 @@ export class BrowserDomExtensionBridge { tabId?: number frameIds?: number[] }) { + const effectiveTimeout = params.timeoutMs ?? this.config.requestTimeoutMs + // NOTICE: The bridge-level timeout must exceed the background-level polling + // timeout, otherwise the bridge rejects before the extension finishes polling. + // The extension can overrun by one full frame send timeout (8s) plus the + // polling interval (500ms), so keep headroom for slow or unresponsive frames. return await this.callAction>>>('waitForElement', { selector: params.selector, - timeoutMs: params.timeoutMs ?? this.config.requestTimeoutMs, + timeoutMs: effectiveTimeout, tabId: params.tabId, frameIds: params.frameIds, + }, { + timeoutMs: effectiveTimeout + WAIT_FOR_ELEMENT_BRIDGE_TIMEOUT_BUFFER_MS, }) }