feat(computer-use-mcp): add read-only DOM tools parity to extension bridge (#1733)

Co-authored-by-agent: Antigravity <antigravity@gemini.com>
2026-04-27 15:24:27 +08:00
parent 0c8c99dae3
commit e145ce81c1
4 changed files with 443 additions and 162 deletions
@@ -28,6 +28,8 @@ const AIRI_BRIDGE_HELLO = {
 }
 const BRIDGE_RECONNECT_MIN_MS = 1_000
 const BRIDGE_RECONNECT_MAX_MS = 10_000
+const SEND_CU_ACTION_TIMEOUT_MS = 8_000
+const WAIT_FOR_ELEMENT_POLL_INTERVAL_MS = 500

 let bridgeSocket = null
 let bridgeReconnectDelayMs = BRIDGE_RECONNECT_MIN_MS
@@ -182,7 +184,7 @@ async function sendCUAction(tabId, frameId, method, args) {
  return new Promise((resolve) => {
    const timeout = setTimeout(() => {
      resolve({ success: false, error: 'sendMessage timeout' })
-    }, 8000)
+    }, SEND_CU_ACTION_TIMEOUT_MS)

    try {
      chrome.tabs.sendMessage(
@@ -400,6 +402,9 @@ async function readAllFramesDOMWithOffsets(tabId, frameIds, opts) {
 * - findElements: find multiple elements by CSS selector
 * - getClickTarget: get center point of an element for click targeting
 * - getElementAttributes: get all attributes of an element
+ * - readInputValue: read the current value of an input/textarea/select
+ * - getComputedStyles: read computed CSS styles of an element
+ * - waitForElement: poll until a CSS selector matches in any frame
 */
 async function handleCommand(cmd) {
  const { action, id } = cmd
@@ -441,6 +446,91 @@ async function handleCommand(cmd) {
        result = await runCUAction(tabId, cmd.frameIds || null, 'getElementAttributes', [cmd.selector || ''])
        break

+      case 'readInputValue':
+        result = await runCUAction(tabId, cmd.frameIds || null, 'readInputValue', [cmd.selector || ''])
+        break
+
+      case 'getComputedStyles':
+        result = await runCUAction(tabId, cmd.frameIds || null, 'getComputedStyles', [cmd.selector || '', cmd.properties || null])
+        break
+
+      case 'waitForElement': {
+        // Background-level polling: repeatedly call findElements until a match
+        // is found or the timeout expires. No DOM MutationObserver needed.
+        const selector = cmd.selector || ''
+        const timeoutMs = Math.min(Math.max(Number(cmd.timeoutMs) || 10_000, 500), 30_000)
+        const deadline = Date.now() + timeoutMs
+        let lastFrames = []
+        let lastPollError = ''
+        let lastFrameError = ''
+
+        result = await new Promise((resolve) => {
+          async function poll() {
+            try {
+              const frames = await runCUAction(tabId, cmd.frameIds || null, 'findElements', [selector, 1])
+              lastFrames = frames
+              const frameErrors = frames
+                .map(entry => unwrapBridgePayload(entry.result))
+                .filter(payload => payload && payload.success === false && typeof payload.error === 'string')
+                .map(payload => payload.error)
+              if (frameErrors.length > 0) {
+                lastFrameError = frameErrors.join('; ')
+              }
+
+              const found = frames.some((entry) => {
+                const payload = unwrapBridgePayload(entry.result)
+                return payload && payload.success && Array.isArray(payload.elements) && payload.elements.length > 0
+              })
+              if (found) {
+                resolve(frames)
+                return
+              }
+            }
+            catch (e) {
+              lastPollError = e?.message || String(e)
+            }
+
+            if (Date.now() >= deadline) {
+              if (lastFrames.length === 0) {
+                let frameIds = []
+                if (Array.isArray(cmd.frameIds) && cmd.frameIds.length > 0) {
+                  frameIds = cmd.frameIds
+                }
+                else if (typeof cmd.frameIds === 'number') {
+                  frameIds = [cmd.frameIds]
+                }
+                else {
+                  try {
+                    const frames = await chrome.webNavigation.getAllFrames({ tabId })
+                    frameIds = frames.map(frame => frame.frameId)
+                  }
+                  catch {
+                    frameIds = [0]
+                  }
+                }
+                lastFrames = frameIds.map(frameId => ({ frameId }))
+              }
+
+              const lastError = lastPollError || lastFrameError || undefined
+              resolve(lastFrames.map(entry => ({
+                frameId: entry.frameId,
+                result: {
+                  success: false,
+                  error: `timed out waiting for selector "${selector}"`,
+                  selector,
+                  timeoutMs,
+                  ...(lastError ? { lastError } : {}),
+                },
+              })))
+              return
+            }
+            setTimeout(poll, WAIT_FOR_ELEMENT_POLL_INTERVAL_MS)
+          }
+          poll()
+        })
+        break
+      }
+
      default:
        return { id, ok: false, error: `unknown action: ${action}` }
    }
@@ -243,6 +243,98 @@
        return { success: false, error: e.message }
      }
    },
+
+    /**
+     * Read the current value of an input, textarea, or select element.
+     * Returns value plus basic element metadata. Read-only: no DOM mutation.
+     */
+    readInputValue(selector) {
+      try {
+        const el = document.querySelector(selector)
+        if (!el)
+          return { success: false, error: 'not found' }
+
+        const tag = el.tagName.toLowerCase()
+        if (tag !== 'input' && tag !== 'textarea' && tag !== 'select')
+          return { success: false, error: 'element is not an input, textarea, or select' }
+
+        const type = typeof el.type === 'string' ? el.type : ''
+        const rawValue = String(el.value ?? '')
+        const isPassword = tag === 'input' && type.toLowerCase() === 'password'
+        const result = {
+          value: isPassword ? '[redacted]' : rawValue.slice(0, 60),
+          valueLength: rawValue.length,
+          valueRedacted: isPassword,
+          valueTruncated: !isPassword && rawValue.length > 60,
+          tag,
+          id: el.id || '',
+          name: el.name || '',
+          type,
+        }
+
+        if (tag === 'input' && (type === 'checkbox' || type === 'radio')) {
+          result.checked = !!el.checked
+        }
+
+        if (tag === 'select') {
+          result.selectedIndex = el.selectedIndex
+          result.selectedText = el.selectedIndex >= 0 && el.options[el.selectedIndex]
+            ? el.options[el.selectedIndex].text.slice(0, 120)
+            : ''
+        }
+
+        return { success: true, ...result }
+      }
+      catch (e) {
+        return { success: false, error: e.message }
+      }
+    },
+
+    /**
+     * Read computed CSS styles of an element.
+     * If `properties` is provided, return only those properties.
+     * Otherwise return a controlled default set to avoid dumping the full
+     * CSSStyleDeclaration (which can be multi-KB).
+     */
+    getComputedStyles(selector, properties) {
+      const DEFAULT_PROPERTIES = [
+        'display',
+        'visibility',
+        'opacity',
+        'position',
+        'width',
+        'height',
+        'color',
+        'background-color',
+        'font-size',
+        'font-family',
+        'overflow',
+        'z-index',
+        'pointer-events',
+        'cursor',
+      ]
+
+      try {
+        const el = document.querySelector(selector)
+        if (!el)
+          return { success: false, error: 'not found' }
+
+        const computed = window.getComputedStyle(el)
+        const props = Array.isArray(properties) && properties.length > 0
+          ? properties
+          : DEFAULT_PROPERTIES
+        const styles = Object.create(null)
+
+        for (const prop of props) {
+          styles[prop] = computed.getPropertyValue(prop)
+        }
+
+        return { success: true, styles }
+      }
+      catch (e) {
+        return { success: false, error: e.message }
+      }
+    },
  }

  window.__AIRI_DG__ = __AIRI_DG__
@@ -3,6 +3,73 @@ import { WebSocket, WebSocketServer } from 'ws'

 import { BrowserDomExtensionBridge } from './extension-bridge'

+/**
+ * Helper: create a bridge + client pair, wait for the client to connect
+ * and send the hello handshake.
+ */
+async function createConnectedBridge(config?: Partial<{
+  requestTimeoutMs: number
+}>) {
+  const bridge = new BrowserDomExtensionBridge({
+    enabled: true,
+    host: '127.0.0.1',
+    port: 0,
+    requestTimeoutMs: config?.requestTimeoutMs ?? 1_000,
+  })
+  await bridge.start()
+
+  const status = bridge.getStatus()
+  const client = new WebSocket(`ws://${status.host}:${status.port}`)
+
+  await new Promise<void>((resolve, reject) => {
+    client.once('open', () => {
+      client.send(JSON.stringify({
+        type: 'hello',
+        source: 'test-extension',
+        version: 'bridge-test',
+      }))
+      resolve()
+    })
+    client.once('error', reject)
+  })
+
+  return { bridge, client }
+}
+
+/**
+ * Helper: register a mock handler on the client that echoes a fixed
+ * result for a given action name.
+ */
+function mockClientAction(
+  client: WebSocket,
+  actionName: string,
+  resultFn: (data: Record<string, unknown>) => unknown,
+  opts?: { delayMs?: number },
+) {
+  client.on('message', (raw) => {
+    const data = JSON.parse(String(raw)) as Record<string, unknown>
+    if (typeof data.id !== 'string')
+      return
+    if (data.action !== actionName)
+      return
+
+    const respond = () => {
+      client.send(JSON.stringify({
+        id: data.id,
+        ok: true,
+        result: resultFn(data),
+      }))
+    }
+
+    if (opts?.delayMs) {
+      setTimeout(respond, opts.delayMs)
+    }
+    else {
+      respond()
+    }
+  })
+}
+
 describe('browserDomExtensionBridge', () => {
  let bridge: BrowserDomExtensionBridge | undefined
  let client: WebSocket | undefined
@@ -22,45 +89,14 @@ describe('browserDomExtensionBridge', () => {
  })

  it('round-trips actions over the extension websocket bridge', async () => {
-    bridge = new BrowserDomExtensionBridge({
-      enabled: true,
-      host: '127.0.0.1',
-      port: 0,
-      requestTimeoutMs: 1_000,
-    })
-    await bridge.start()
+    const result = await createConnectedBridge()
+    bridge = result.bridge
+    client = result.client

-    const status = bridge.getStatus()
-    client = new WebSocket(`ws://${status.host}:${status.port}`)
-
-    client.on('message', (raw) => {
-      const data = JSON.parse(String(raw)) as Record<string, unknown>
-      if (typeof data.id !== 'string')
-        return
-
-      if (data.action === 'getActiveTab') {
-        client!.send(JSON.stringify({
-          id: data.id,
-          ok: true,
-          result: {
-            title: 'AIRI Demo Tab',
-            url: 'https://example.com/demo',
-          },
-        }))
-      }
-    })
-
-    await new Promise<void>((resolve, reject) => {
-      client!.once('open', () => {
-        client!.send(JSON.stringify({
-          type: 'hello',
-          source: 'test-extension',
-          version: 'bridge-test',
-        }))
-        resolve()
-      })
-      client!.once('error', reject)
-    })
+    mockClientAction(client, 'getActiveTab', () => ({
+      title: 'AIRI Demo Tab',
+      url: 'https://example.com/demo',
+    }))

    const activeTab = await bridge.getActiveTab()

@@ -72,98 +108,183 @@ describe('browserDomExtensionBridge', () => {
    expect(bridge.getStatus().lastHello?.source).toBe('test-extension')
  })

-  it('rejects clickSelector on the read-only extension transport even when getClickTarget succeeds', async () => {
+  it('supportsAction returns true for read-only actions and false for mutating actions', () => {
    bridge = new BrowserDomExtensionBridge({
      enabled: true,
      host: '127.0.0.1',
      port: 0,
      requestTimeoutMs: 1_000,
    })
-    await bridge.start()

-    const status = bridge.getStatus()
-    client = new WebSocket(`ws://${status.host}:${status.port}`)
+    // Read-only actions should be supported
+    expect(bridge.supportsAction('readInputValue')).toBe(true)
+    expect(bridge.supportsAction('getComputedStyles')).toBe(true)
+    expect(bridge.supportsAction('waitForElement')).toBe(true)
+    expect(bridge.supportsAction('getActiveTab')).toBe(true)
+    expect(bridge.supportsAction('findElements')).toBe(true)
+    expect(bridge.supportsAction('getElementAttributes')).toBe(true)

-    client.on('message', (raw) => {
-      const data = JSON.parse(String(raw)) as Record<string, unknown>
-      if (typeof data.id !== 'string')
-        return
-
-      if (data.action === 'getClickTarget') {
-        client!.send(JSON.stringify({
-          id: data.id,
-          ok: true,
-          result: [
-            {
-              frameId: 5,
-              result: {
-                success: true,
-                x: 321,
-                y: 182,
-                element: {
-                  tag: 'button',
-                  text: 'Submit',
-                },
-                center: {
-                  x: 321,
-                  y: 182,
-                },
-              },
-            },
-          ],
-        }))
-        return
-      }
-
-      if (data.action === 'clickAt') {
-        client!.send(JSON.stringify({
-          id: data.id,
-          ok: true,
-          result: [
-            {
-              frameId: 5,
-              result: {
-                success: true,
-              },
-            },
-          ],
-        }))
-      }
-    })
-
-    await new Promise<void>((resolve, reject) => {
-      client!.once('open', () => {
-        client!.send(JSON.stringify({
-          type: 'hello',
-          source: 'test-extension',
-          version: 'bridge-test',
-        }))
-        resolve()
-      })
-      client!.once('error', reject)
-    })
-
-    await expect(bridge.clickSelector({
-      selector: '#submit',
-      frameIds: [5],
-    })).rejects.toThrow('does not support action "clickAt"')
+    // Mutating actions should NOT be supported
+    expect(bridge.supportsAction('setInputValue')).toBe(false)
+    expect(bridge.supportsAction('checkCheckbox')).toBe(false)
+    expect(bridge.supportsAction('selectOption')).toBe(false)
+    expect(bridge.supportsAction('triggerEvent')).toBe(false)
+    expect(bridge.supportsAction('clickAt')).toBe(false)
  })

-  it('rejects unsupported DOM-mutating actions before sending them to the extension transport', async () => {
-    bridge = new BrowserDomExtensionBridge({
-      enabled: true,
-      host: '127.0.0.1',
-      port: 0,
-      requestTimeoutMs: 1_000,
-    })
-    await bridge.start()
+  it('readInputValue round-trips through the bridge', async () => {
+    const result = await createConnectedBridge()
+    bridge = result.bridge
+    client = result.client

-    expect(bridge.supportsAction('readAllFramesDOM')).toBe(true)
-    expect(bridge.supportsAction('setInputValue')).toBe(false)
-    await expect(bridge.setInputValue({
-      selector: '#email',
-      value: 'hello@example.com',
-    })).rejects.toThrow('does not support action "setInputValue"')
+    mockClientAction(client, 'readInputValue', () => ([
+      {
+        frameId: 0,
+        result: {
+          success: true,
+          value: 'hello world',
+          tag: 'input',
+          id: 'search',
+          name: 'q',
+          type: 'text',
+        },
+      },
+    ]))
+
+    const frames = await bridge.readInputValue({ selector: '#search' })
+
+    expect(frames).toEqual([
+      {
+        frameId: 0,
+        result: {
+          success: true,
+          value: 'hello world',
+          tag: 'input',
+          id: 'search',
+          name: 'q',
+          type: 'text',
+        },
+      },
+    ])
+  })
+
+  it('getComputedStyles round-trips through the bridge', async () => {
+    const result = await createConnectedBridge()
+    bridge = result.bridge
+    client = result.client
+
+    mockClientAction(client, 'getComputedStyles', () => ([
+      {
+        frameId: 0,
+        result: {
+          success: true,
+          styles: {
+            display: 'block',
+            visibility: 'visible',
+            opacity: '1',
+          },
+        },
+      },
+    ]))
+
+    const frames = await bridge.getComputedStyles({
+      selector: '.container',
+      properties: ['display', 'visibility', 'opacity'],
+    })
+
+    expect(frames).toEqual([
+      {
+        frameId: 0,
+        result: {
+          success: true,
+          styles: {
+            display: 'block',
+            visibility: 'visible',
+            opacity: '1',
+          },
+        },
+      },
+    ])
+  })
+
+  it('waitForElement uses action-specific timeout, not default requestTimeoutMs', async () => {
+    // Configure bridge with a very short default timeout (500ms)
+    const result = await createConnectedBridge({ requestTimeoutMs: 500 })
+    bridge = result.bridge
+    client = result.client
+
+    // Mock: respond after 800ms — longer than the 500ms default but well
+    // within the 3000ms action-specific timeout we'll pass.
+    mockClientAction(client, 'waitForElement', () => ([
+      {
+        frameId: 0,
+        result: { success: true, elements: [{ tag: 'div', id: 'lazy' }] },
+      },
+    ]), { delayMs: 800 })
+
+    // This should NOT reject at 500ms because waitForElement uses a
+    // bridge-level timeout override that covers extension-side polling.
+    const frames = await bridge.waitForElement({
+      selector: '#lazy',
+      timeoutMs: 3_000,
+    })
+
+    expect(frames).toEqual([
+      {
+        frameId: 0,
+        result: { success: true, elements: [{ tag: 'div', id: 'lazy' }] },
+      },
+    ])
+  })
+
+  it('waitForElement uses the default requestTimeoutMs for extension-side polling when no timeoutMs is provided', async () => {
+    const result = await createConnectedBridge({ requestTimeoutMs: 200 })
+    bridge = result.bridge
+    client = result.client
+
+    mockClientAction(client, 'waitForElement', data => ([
+      {
+        frameId: 0,
+        result: {
+          success: false,
+          selector: data.selector,
+          timeoutMs: data.timeoutMs,
+          error: 'timed out waiting for selector "#missing"',
+        },
+      },
+    ]), { delayMs: 300 })
+
+    const frames = await bridge.waitForElement({ selector: '#missing' })
+
+    expect(frames).toEqual([
+      {
+        frameId: 0,
+        result: {
+          success: false,
+          selector: '#missing',
+          timeoutMs: 200,
+          error: 'timed out waiting for selector "#missing"',
+        },
+      },
+    ])
+  })
+
+  it('rejects pending requests when the bridge disconnects', async () => {
+    const result = await createConnectedBridge()
+    bridge = result.bridge
+    client = result.client
+
+    // Start a request but don't respond to it
+    const promise = bridge.getActiveTab()
+    expect(bridge.getStatus().pendingRequests).toBe(1)
+
+    // Close the client to simulate disconnection
+    client.close()
+    client = undefined
+
+    await expect(promise).rejects.toThrow(/disconnected/)
+    expect(bridge.getStatus().connected).toBe(false)
+    expect(bridge.getStatus().pendingRequests).toBe(0)
  })

  it('can retry startup after an initial bind failure', async () => {
@@ -213,43 +334,4 @@ describe('browserDomExtensionBridge', () => {
    await new Promise(resolve => setTimeout(resolve, 10))
    expect(bridge.getStatus().connected).toBe(true)
  })
-
-  it('rejects in-flight requests immediately when the socket disconnects', async () => {
-    bridge = new BrowserDomExtensionBridge({
-      enabled: true,
-      host: '127.0.0.1',
-      port: 0,
-      requestTimeoutMs: 10_000,
-    })
-    await bridge.start()
-
-    const status = bridge.getStatus()
-    client = new WebSocket(`ws://${status.host}:${status.port}`)
-
-    client.on('message', (raw) => {
-      const data = JSON.parse(String(raw)) as Record<string, unknown>
-      if (data.action === 'getActiveTab') {
-        client!.close()
-      }
-    })
-
-    await new Promise<void>((resolve, reject) => {
-      client!.once('open', () => {
-        client!.send(JSON.stringify({
-          type: 'hello',
-          source: 'test-extension',
-          version: 'bridge-test',
-        }))
-        resolve()
-      })
-      client!.once('error', reject)
-    })
-
-    const startedAt = Date.now()
-
-    await expect(bridge.getActiveTab()).rejects.toThrow('browser dom bridge disconnected before completing pending request')
-    expect(Date.now() - startedAt).toBeLessThan(1_000)
-    expect(bridge.getStatus().pendingRequests).toBe(0)
-    expect(bridge.getStatus().connected).toBe(false)
-  })
 })
@@ -19,7 +19,11 @@ const SUPPORTED_ACTIONS = new Set([
  'findElements',
  'getClickTarget',
  'getElementAttributes',
+  'readInputValue',
+  'getComputedStyles',
+  'waitForElement',
 ])
+const WAIT_FOR_ELEMENT_BRIDGE_TIMEOUT_BUFFER_MS = 9_500

 interface PendingBridgeRequest {
  reject: (error: Error) => void
@@ -176,7 +180,11 @@ export class BrowserDomExtensionBridge {
    }
  }

-  async callAction<TResult = unknown>(action: string, payload: Record<string, unknown> = {}): Promise<TResult> {
+  async callAction<TResult = unknown>(
+    action: string,
+    payload: Record<string, unknown> = {},
+    options?: { timeoutMs?: number },
+  ): Promise<TResult> {
    if (!this.config.enabled) {
      throw new Error('browser dom bridge is disabled')
    }
@@ -196,12 +204,14 @@ export class BrowserDomExtensionBridge {
      ...payload,
    }

+    const effectiveTimeoutMs = options?.timeoutMs ?? this.config.requestTimeoutMs
+
    const result = await new Promise<TResult>((resolve, reject) => {
      const timeoutId = setTimeout(() => {
        this.pending.delete(id)
        this.status.pendingRequests = this.pending.size
        reject(new Error(`browser dom bridge timed out waiting for ${action}`))
-      }, this.config.requestTimeoutMs)
+      }, effectiveTimeoutMs)

      this.pending.set(id, {
        resolve: value => resolve(value as TResult),
@@ -364,11 +374,18 @@ export class BrowserDomExtensionBridge {
    tabId?: number
    frameIds?: number[]
  }) {
+    const effectiveTimeout = params.timeoutMs ?? this.config.requestTimeoutMs
+    // NOTICE: The bridge-level timeout must exceed the background-level polling
+    // timeout, otherwise the bridge rejects before the extension finishes polling.
+    // The extension can overrun by one full frame send timeout (8s) plus the
+    // polling interval (500ms), so keep headroom for slow or unresponsive frames.
    return await this.callAction<Array<BrowserDomFrameResult<Record<string, unknown>>>>('waitForElement', {
      selector: params.selector,
-      timeoutMs: params.timeoutMs ?? this.config.requestTimeoutMs,
+      timeoutMs: effectiveTimeout,
      tabId: params.tabId,
      frameIds: params.frameIds,
+    }, {
+      timeoutMs: effectiveTimeout + WAIT_FOR_ELEMENT_BRIDGE_TIMEOUT_BUFFER_MS,
    })
  }