From 746e486628dae8057b634c71cde95f7c725e34a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A2=93=E6=81=92?= <160735726+3361559784@users.noreply.github.com> Date: Wed, 13 May 2026 12:16:39 +0800 Subject: [PATCH] feat(computer-use-mcp): add background desktop scheduler (#1805) --- .../src/chrome-semantic-adapter.ts | 2 +- .../src/desktop-grounding-capture.test.ts | 107 +++++++++ .../src/desktop-grounding-types.ts | 2 +- .../src/desktop-grounding.test.ts | 17 ++ .../computer-use-mcp/src/desktop-grounding.ts | 87 ++++--- .../src/server/action-executor.test.ts | 215 ++++++++++++++++++ .../src/server/desktop-grounding-actions.ts | 88 +++++-- .../src/server/desktop-scheduler.test.ts | 83 +++++++ .../src/server/desktop-scheduler.ts | 118 ++++++++++ .../register-desktop-grounding-tools.test.ts | 27 +++ .../src/server/register-desktop-grounding.ts | 39 +--- 11 files changed, 699 insertions(+), 86 deletions(-) create mode 100644 services/computer-use-mcp/src/desktop-grounding-capture.test.ts create mode 100644 services/computer-use-mcp/src/server/desktop-scheduler.test.ts create mode 100644 services/computer-use-mcp/src/server/desktop-scheduler.ts diff --git a/services/computer-use-mcp/src/chrome-semantic-adapter.ts b/services/computer-use-mcp/src/chrome-semantic-adapter.ts index e127b9b72..a3baff866 100644 --- a/services/computer-use-mcp/src/chrome-semantic-adapter.ts +++ b/services/computer-use-mcp/src/chrome-semantic-adapter.ts @@ -3,7 +3,7 @@ * and maps it to DesktopTargetCandidate format. * * Uses the extension bridge as primary source and CDP bridge as fallback. - * Only active when Chrome is the foreground app. + * Best effort when browser surfaces are available. * * The adapter handles coordinate transformation from page-relative * (CSS viewport) coordinates to screen-absolute coordinates using diff --git a/services/computer-use-mcp/src/desktop-grounding-capture.test.ts b/services/computer-use-mcp/src/desktop-grounding-capture.test.ts new file mode 100644 index 000000000..c5ad1917c --- /dev/null +++ b/services/computer-use-mcp/src/desktop-grounding-capture.test.ts @@ -0,0 +1,107 @@ +import type { AXSnapshot } from './accessibility/types' +import type { DesktopExecutor, WindowObservation } from './types' + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { captureDesktopGrounding } from './desktop-grounding' + +const { captureAXTreeMock, captureChromeSemanticsMock } = vi.hoisted(() => ({ + captureAXTreeMock: vi.fn(), + captureChromeSemanticsMock: vi.fn(), +})) + +vi.mock('./accessibility', () => ({ + captureAXTree: captureAXTreeMock, +})) + +vi.mock('./chrome-semantic-adapter', async () => { + const actual = await vi.importActual('./chrome-semantic-adapter') + return { + ...actual, + captureChromeSemantics: captureChromeSemanticsMock, + } +}) + +function makeAxSnapshot(): AXSnapshot { + const root = { + uid: 'root', + role: 'AXApplication', + title: 'AIRI', + children: [], + } + + return { + snapshotId: 'ax_1', + pid: 123, + appName: 'AIRI', + root, + uidToNode: new Map([['root', root]]), + capturedAt: new Date().toISOString(), + maxDepth: 1, + truncated: false, + } as AXSnapshot +} + +describe('captureDesktopGrounding', () => { + beforeEach(() => { + captureAXTreeMock.mockReset() + captureChromeSemanticsMock.mockReset() + }) + + it('does not project Chrome semantics onto non-Chrome windows that only share the same title', async () => { + captureAXTreeMock.mockResolvedValue(makeAxSnapshot()) + captureChromeSemanticsMock.mockResolvedValue({ + pageUrl: 'https://example.com', + pageTitle: 'Shared Title', + interactiveElements: [ + { + tag: 'button', + text: 'Submit', + rect: { x: 20, y: 20, w: 80, h: 30 }, + }, + ], + capturedAt: new Date().toISOString(), + source: 'extension', + }) + + const genericObservation: WindowObservation = { + frontmostAppName: 'AIRI', + windows: [ + { + id: 'airi:1', + appName: 'AIRI', + title: 'Shared Title', + bounds: { x: 10, y: 20, width: 1200, height: 800 }, + }, + ], + observedAt: new Date().toISOString(), + } + + const chromeObservation: WindowObservation = { + windows: [], + observedAt: new Date().toISOString(), + } + + const observeWindows = vi.fn() + .mockResolvedValueOnce(genericObservation) + .mockResolvedValueOnce(chromeObservation) + + const executor = { + takeScreenshot: vi.fn().mockResolvedValue({ + dataBase64: '', + mimeType: 'image/png', + path: '', + capturedAt: new Date().toISOString(), + }), + observeWindows, + } as unknown as DesktopExecutor + + const snapshot = await captureDesktopGrounding({ + config: {} as never, + executor, + input: { includeChrome: true }, + }) + + expect(snapshot.targetCandidates.some(candidate => candidate.source === 'chrome_dom')).toBe(false) + }) +}) diff --git a/services/computer-use-mcp/src/desktop-grounding-types.ts b/services/computer-use-mcp/src/desktop-grounding-types.ts index 63408aca9..ab333471f 100644 --- a/services/computer-use-mcp/src/desktop-grounding-types.ts +++ b/services/computer-use-mcp/src/desktop-grounding-types.ts @@ -147,7 +147,7 @@ export interface DesktopGroundingSnapshot { screenshot: ScreenshotArtifact /** macOS AX tree snapshot (if captured successfully) */ axSnapshot?: AXSnapshot - /** Chrome semantic snapshot (only when Chrome is foreground) */ + /** Chrome semantic snapshot (best effort when browser surfaces are available) */ chromeSemanticSnapshot?: ChromeSemanticSnapshot /** Merged, deduplicated, ranked target candidates */ targetCandidates: DesktopTargetCandidate[] diff --git a/services/computer-use-mcp/src/desktop-grounding.test.ts b/services/computer-use-mcp/src/desktop-grounding.test.ts index 26966b260..88715712f 100644 --- a/services/computer-use-mcp/src/desktop-grounding.test.ts +++ b/services/computer-use-mcp/src/desktop-grounding.test.ts @@ -104,6 +104,7 @@ describe('buildTargetCandidates', () => { expect(candidates.length).toBe(2) expect(candidates[0].source).toBe('chrome_dom') expect(candidates[0].tag).toBe('button') + expect(candidates[0].appName).toBe('Google Chrome') }) it('chrome + AX: deduplicates overlapping candidates', () => { @@ -168,6 +169,22 @@ describe('buildTargetCandidates', () => { const candidates = buildTargetCandidates({ axSnapshot: ax, foregroundApp: 'Finder' }) expect(candidates[0].interactable).toBe(false) }) + + it('keeps chrome_dom candidates attached to Google Chrome even when another app is foreground', () => { + const chrome = makeChromeSnapshot([ + { tag: 'button', text: 'Submit', rect: { x: 10, y: 10, w: 80, h: 30 } }, + ]) + + const candidates = buildTargetCandidates({ + chromeSnapshot: chrome, + chromeWindowBounds: { x: 0, y: 0, width: 1920, height: 1080 }, + foregroundApp: 'Finder', + }) + + expect(candidates).toHaveLength(1) + expect(candidates[0].source).toBe('chrome_dom') + expect(candidates[0].appName).toBe('Google Chrome') + }) }) describe('captureDesktopGrounding', () => { diff --git a/services/computer-use-mcp/src/desktop-grounding.ts b/services/computer-use-mcp/src/desktop-grounding.ts index 0f60a634d..40518a853 100644 --- a/services/computer-use-mcp/src/desktop-grounding.ts +++ b/services/computer-use-mcp/src/desktop-grounding.ts @@ -3,8 +3,8 @@ * * This is the main entry point for the `desktop_observe` tool. * It captures screenshot, window observation, AX tree, and Chrome semantics - * in parallel, then merges everything into a single `DesktopGroundingSnapshot` - * with deduplicated, ranked target candidates. + * in parallel when available, then merges everything into a single + * `DesktopGroundingSnapshot` with deduplicated, ranked target candidates. */ import type { AXNode, AXSnapshot } from './accessibility/types' @@ -36,21 +36,13 @@ import { boundsIoU } from './snap-resolver' */ const STALENESS_THRESHOLD_MS = 2000 -/** Known Chrome-like browser app names (lowercase, no .app suffix) */ -const CHROME_APPS = new Set([ - 'google chrome', - 'chrome', - 'google chrome canary', - 'chromium', -]) - let nextSnapshotId = 1 /** * Capture a unified desktop grounding snapshot. * * Runs screenshot, window observation, and AX tree capture in parallel. - * If the foreground app is Chrome (and `includeChrome` is not false), + * If Chrome browser surfaces are available (and `includeChrome` is not false), * also captures Chrome semantic data. * * @param params - Capture parameters (config, executor, input, bridges) @@ -79,7 +71,27 @@ export async function captureDesktopGrounding(params: { // Determine foreground app const foregroundApp = windowObs.frontmostAppName || axSnapshot?.appName || 'unknown' - const isChromeInFront = isChromeApp(foregroundApp) + const shouldCaptureChrome = input?.includeChrome !== false + + // Ask the executor for a Chrome-filtered window list when Chrome semantics + // are requested. The generic top-N window snapshot is often dominated by + // system UI and can miss Chrome entirely, which would prevent chrome_dom + // candidates from being mapped to screen coordinates. + let chromeWindowBounds = findChromeWindowBounds(windowObs) + let chromeWindowObservation: WindowObservation | undefined + if (shouldCaptureChrome && !chromeWindowBounds) { + try { + chromeWindowObservation = await executor.observeWindows({ + app: 'Google Chrome', + limit: 12, + }) + chromeWindowBounds = findChromeWindowBounds(chromeWindowObservation) + } + catch { + // Best-effort only. Fall back to AX-only candidates if filtered window + // enumeration fails. + } + } // If Chrome is foreground, ask the executor for a Chrome-filtered window list. // The generic top-N window snapshot is often dominated by system UI and can @@ -102,8 +114,14 @@ export async function captureDesktopGrounding(params: { // Phase 2: Chrome semantic data (only if Chrome is foreground and allowed) let chromeSemanticSnapshot: ChromeSemanticSnapshot | null = null - if (isChromeInFront && input?.includeChrome !== false) { + if (shouldCaptureChrome) { chromeSemanticSnapshot = await captureChromeSemantics(extensionBridge, cdpBridge) + if (!chromeWindowBounds && chromeSemanticSnapshot?.pageTitle) { + chromeWindowBounds = findChromeWindowBounds( + chromeWindowObservation ?? windowObs, + chromeSemanticSnapshot.pageTitle, + ) + } } // Phase 3: Build target candidates @@ -120,7 +138,7 @@ export async function captureDesktopGrounding(params: { screenshot, axSnapshot, chromeSemanticSnapshot: chromeSemanticSnapshot ?? undefined, - isChromeInFront, + chromeSemanticEnabled: shouldCaptureChrome, assemblyTimestamp: now, }) @@ -162,10 +180,6 @@ export function buildTargetCandidates(params: { chromeSnapshot.interactiveElements, chromeWindowBounds, ) - // Set appName on all chrome candidates - for (const c of chromeCandidates) { - c.appName = foregroundApp - } } // 2. Build AX candidates @@ -329,17 +343,32 @@ function axNodesToTargetCandidates( // Helpers // --------------------------------------------------------------------------- -/** Compiled regex for stripping .app suffix from macOS app names */ -const APP_SUFFIX_RE = /\.app$/u - -function isChromeApp(appName: string): boolean { - return CHROME_APPS.has(appName.trim().toLowerCase().replace(APP_SUFFIX_RE, '')) -} - function findChromeWindowBounds( observation: WindowObservation, - _foregroundApp: string, + titleHint?: string, ): Bounds | undefined { + const normalizedTitleHint = titleHint?.trim().toLowerCase() + if (normalizedTitleHint) { + const titleMatchedWindow = observation.windows.find((window) => { + if (!window.bounds || !window.appName.toLowerCase().includes('chrome')) { + return false + } + + const normalizedWindowTitle = window.title?.trim().toLowerCase() + if (!normalizedWindowTitle) { + return false + } + + return normalizedWindowTitle === normalizedTitleHint + || normalizedWindowTitle.includes(normalizedTitleHint) + || normalizedTitleHint.includes(normalizedWindowTitle) + }) + + if (titleMatchedWindow?.bounds) { + return titleMatchedWindow.bounds + } + } + const chromeWindow = observation.windows.find(w => w.appName.toLowerCase().includes('chrome') && w.bounds, ) @@ -350,10 +379,10 @@ function computeStaleness(params: { screenshot: ScreenshotArtifact axSnapshot?: AXSnapshot chromeSemanticSnapshot?: ChromeSemanticSnapshot - isChromeInFront: boolean + chromeSemanticEnabled: boolean assemblyTimestamp: number }): GroundingStalenessFlags { - const { screenshot, axSnapshot, chromeSemanticSnapshot, isChromeInFront, assemblyTimestamp } = params + const { screenshot, axSnapshot, chromeSemanticSnapshot, chromeSemanticEnabled, assemblyTimestamp } = params const screenshotStale = !screenshot.capturedAt || (assemblyTimestamp - new Date(screenshot.capturedAt).getTime()) > STALENESS_THRESHOLD_MS @@ -363,7 +392,7 @@ function computeStaleness(params: { || !axSnapshot.capturedAt || (assemblyTimestamp - new Date(axSnapshot.capturedAt).getTime()) > STALENESS_THRESHOLD_MS - const chromeStale = !isChromeInFront + const chromeStale = !chromeSemanticEnabled || !chromeSemanticSnapshot || !chromeSemanticSnapshot.capturedAt || (assemblyTimestamp - new Date(chromeSemanticSnapshot.capturedAt).getTime()) > STALENESS_THRESHOLD_MS diff --git a/services/computer-use-mcp/src/server/action-executor.test.ts b/services/computer-use-mcp/src/server/action-executor.test.ts index 1bb1ab98c..44775cd2f 100644 --- a/services/computer-use-mcp/src/server/action-executor.test.ts +++ b/services/computer-use-mcp/src/server/action-executor.test.ts @@ -71,6 +71,9 @@ function createRuntimeForActionTest(configOverrides: Partial connected: true, pendingRequests: 0, }), + supportsAction: vi.fn().mockReturnValue(true), + clickSelector: vi.fn().mockResolvedValue(undefined), + checkCheckbox: vi.fn().mockResolvedValue(undefined), } const cdpBridgeManager = { probeAvailability: vi.fn().mockResolvedValue({ @@ -323,6 +326,218 @@ describe('createExecuteAction', () => { })) }) + it('does not refocus when desktop_click_target stays on browser_dom', async () => { + const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest() + stateManager.updateGroundingSnapshot({ + snapshotId: 'dg_1', + capturedAt: new Date().toISOString(), + foregroundApp: 'Google Chrome', + windows: [], + screenshot: { + dataBase64: '', + mimeType: 'image/png', + path: '', + capturedAt: new Date().toISOString(), + }, + targetCandidates: [ + { + id: 't_0', + source: 'chrome_dom', + appName: 'Google Chrome', + role: 'button', + label: 'Submit', + bounds: { x: 100, y: 200, width: 80, height: 30 }, + confidence: 0.95, + interactable: true, + selector: '#submit', + frameId: 0, + isPageContent: true, + }, + ], + staleFlags: { screenshot: false, ax: false, chromeSemantic: false }, + } as any) + desktopSessionController.getSession.mockReturnValue({ + id: 'ds_1', + controlledApp: 'Google Chrome', + ownedWindows: [], + createdAt: new Date().toISOString(), + lastActiveAt: new Date().toISOString(), + }) + executor.getForegroundContext.mockResolvedValue({ + available: true, + appName: 'AIRI', + platform: 'darwin', + }) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target') + + expect(result.isError).not.toBe(true) + expect(desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled() + expect(executor.focusApp).not.toHaveBeenCalled() + expect(executor.click).not.toHaveBeenCalled() + expect(result.structuredContent).toMatchObject({ + status: 'executed', + backendResult: expect.objectContaining({ + executionMode: 'browser_surface', + }), + }) + }) + + it('refocuses when desktop_click_target falls back to OS input', async () => { + const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest() + stateManager.updateGroundingSnapshot({ + snapshotId: 'dg_1', + capturedAt: new Date().toISOString(), + foregroundApp: 'Google Chrome', + windows: [], + screenshot: { + dataBase64: '', + mimeType: 'image/png', + path: '', + capturedAt: new Date().toISOString(), + }, + targetCandidates: [ + { + id: 't_0', + source: 'ax', + appName: 'Google Chrome', + role: 'AXButton', + label: 'Submit', + bounds: { x: 100, y: 200, width: 80, height: 30 }, + confidence: 0.95, + interactable: true, + }, + ], + staleFlags: { screenshot: false, ax: false, chromeSemantic: false }, + } as any) + desktopSessionController.getSession.mockReturnValue({ + id: 'ds_1', + controlledApp: 'Google Chrome', + ownedWindows: [], + createdAt: new Date().toISOString(), + lastActiveAt: new Date().toISOString(), + }) + desktopSessionController.ensureControlledAppInForeground.mockResolvedValue(true) + executor.getForegroundContext.mockResolvedValue({ + available: true, + appName: 'AIRI', + platform: 'darwin', + }) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target') + + expect(result.isError).not.toBe(true) + expect(desktopSessionController.ensureControlledAppInForeground).toHaveBeenCalled() + expect(executor.click).toHaveBeenCalledOnce() + }) + + it('focuses the candidate app before OS fallback when no controlled session exists', async () => { + const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest() + stateManager.updateGroundingSnapshot({ + snapshotId: 'dg_1', + capturedAt: new Date().toISOString(), + foregroundApp: 'Google Chrome', + windows: [], + screenshot: { + dataBase64: '', + mimeType: 'image/png', + path: '', + capturedAt: new Date().toISOString(), + }, + targetCandidates: [ + { + id: 't_0', + source: 'chrome_dom', + appName: 'Google Chrome', + role: 'button', + label: 'Submit', + bounds: { x: 100, y: 200, width: 80, height: 30 }, + confidence: 0.95, + interactable: true, + selector: '#submit', + frameId: 0, + isPageContent: true, + }, + ], + staleFlags: { screenshot: false, ax: false, chromeSemantic: false }, + } as any) + desktopSessionController.getSession.mockReturnValue(null) + executor.getForegroundContext.mockResolvedValue({ + available: true, + appName: 'Finder', + platform: 'darwin', + }) + ;(runtime.browserDomBridge.supportsAction as any).mockReturnValue(false) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target') + + expect(result.isError).not.toBe(true) + expect(desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled() + expect(executor.focusApp).toHaveBeenCalledWith({ app: 'Google Chrome' }) + expect(executor.click).toHaveBeenCalledOnce() + expect(result.structuredContent).toMatchObject({ + status: 'executed', + backendResult: expect.objectContaining({ + executionRoute: 'os_input', + executionMode: 'foreground', + routeNote: expect.stringContaining('focused Google Chrome before OS input fallback'), + }), + }) + }) + + it('rejects cross-app desktop_click_target fallback under a controlled session', async () => { + const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest() + stateManager.updateGroundingSnapshot({ + snapshotId: 'dg_1', + capturedAt: new Date().toISOString(), + foregroundApp: 'AIRI', + windows: [], + screenshot: { + dataBase64: '', + mimeType: 'image/png', + path: '', + capturedAt: new Date().toISOString(), + }, + targetCandidates: [ + { + id: 't_0', + source: 'ax', + appName: 'AIRI', + role: 'AXButton', + label: 'Submit', + bounds: { x: 100, y: 200, width: 80, height: 30 }, + confidence: 0.95, + interactable: true, + }, + ], + staleFlags: { screenshot: false, ax: false, chromeSemantic: false }, + } as any) + desktopSessionController.getSession.mockReturnValue({ + id: 'ds_1', + controlledApp: 'Google Chrome', + ownedWindows: [], + createdAt: new Date().toISOString(), + lastActiveAt: new Date().toISOString(), + }) + executor.getForegroundContext.mockResolvedValue({ + available: true, + appName: 'AIRI', + platform: 'darwin', + }) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target') + + expect(result.isError).toBe(true) + expect(result.content.find(item => item.type === 'text')?.text).toContain('cross-app fallback') + expect(desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled() + expect(executor.focusApp).not.toHaveBeenCalled() + expect(executor.click).not.toHaveBeenCalled() + }) + it('returns a structured failure when controlled-app refocus fails during desktop_click_target execution', async () => { const { runtime, executor, session, stateManager, desktopSessionController } = createRuntimeForActionTest() stateManager.updateGroundingSnapshot({ diff --git a/services/computer-use-mcp/src/server/desktop-grounding-actions.ts b/services/computer-use-mcp/src/server/desktop-grounding-actions.ts index a45844ba5..0893567d8 100644 --- a/services/computer-use-mcp/src/server/desktop-grounding-actions.ts +++ b/services/computer-use-mcp/src/server/desktop-grounding-actions.ts @@ -3,10 +3,12 @@ import type { ComputerUseServerRuntime } from './runtime' import { errorMessageFrom } from '@moeru/std' +import { appNamesMatch } from '../app-aliases' import { decideBrowserAction } from '../browser-action-router' import { getUnsupportedBrowserDomActions, isBrowserDomActionSupported } from '../browser-dom/capabilities' import { resolveSnapByCandidate } from '../snap-resolver' import { sleep } from '../utils/sleep' +import { decideDesktopExecutionMode } from './desktop-scheduler' const DESKTOP_CLICK_SNAPSHOT_MAX_AGE_MS = 5000 @@ -46,23 +48,6 @@ export async function executeDesktopClickTarget( throw new Error(`Candidate "${candidateId}" not found in snapshot "${snapshot.snapshotId}". Available candidates: ${snapshot.targetCandidates.map(c => c.id).join(', ')}`) } - const sessionCtrl = runtime.desktopSessionController - const activeSession = sessionCtrl.getSession() - if (activeSession?.controlledApp) { - const currentForeground = await runtime.executor.getForegroundContext() - const wasAlreadyInFront = await sessionCtrl.ensureControlledAppInForeground({ - currentForeground, - chromeSessionManager: runtime.chromeSessionManager, - activateApp: async (appName) => { - await runtime.executor.focusApp({ app: appName }) - }, - }) - if (!wasAlreadyInFront) { - await sleep(200) - } - sessionCtrl.touch() - } - const candidate = snapshot.targetCandidates.find(c => c.id === candidateId) const intent = { mode: 'execute' as const, @@ -83,6 +68,9 @@ export async function executeDesktopClickTarget( let routeNote = '' let routeReason = 'candidate not found' let osInputResult: ExecutorActionResult | undefined + let executionMode: 'background' | 'browser_surface' | 'foreground' = 'foreground' + let executionModeReason = 'desktop_click_target uses native input and needs foreground access' + let fallbackForegroundApp: string | undefined const executeOsClick = async () => { const result = await runtime.executor.click({ @@ -96,12 +84,62 @@ export async function executeDesktopClickTarget( return result } + const ensureForegroundForOsInput = async () => { + const sessionCtrl = runtime.desktopSessionController + const activeSession = sessionCtrl.getSession() + if (!activeSession?.controlledApp) { + const candidateApp = candidate?.appName?.trim() + if (!candidateApp || candidateApp === 'unknown') { + throw new Error('desktop_click_target cannot fall back to OS input without a controlled app session or a concrete target app.') + } + + const currentForeground = await runtime.executor.getForegroundContext() + if (currentForeground.available && currentForeground.appName === candidateApp) { + fallbackForegroundApp = candidateApp + return + } + + await runtime.executor.focusApp({ app: candidateApp }) + fallbackForegroundApp = candidateApp + await sleep(200) + return + } + + if (!appNamesMatch(candidate?.appName, activeSession.controlledApp)) { + throw new Error(`desktop_click_target rejected cross-app fallback: candidate "${candidate?.appName ?? 'unknown'}" does not match controlled app "${activeSession.controlledApp}".`) + } + + const currentForeground = await runtime.executor.getForegroundContext() + const wasAlreadyInFront = await sessionCtrl.ensureControlledAppInForeground({ + currentForeground, + chromeSessionManager: runtime.chromeSessionManager, + activateApp: async (appName) => { + await runtime.executor.focusApp({ app: appName }) + }, + }) + fallbackForegroundApp = activeSession.controlledApp + if (!wasAlreadyInFront) { + await sleep(200) + } + sessionCtrl.touch() + } + try { const bridgeConnected = runtime.browserDomBridge?.getStatus().connected ?? false const routeDecision = candidate ? decideBrowserAction(candidate, bridgeConnected, button, clickCount) : { route: 'os_input' as const, reason: 'candidate not found' } + const schedulingDecision = decideDesktopExecutionMode({ + action: { kind: 'desktop_click_target', input }, + browserSurface: runtime.stateManager.getState().browserSurfaceAvailability, + browserDomRoute: routeDecision.route === 'browser_dom', + }) + if (routeDecision.route === 'browser_dom') { + executionMode = schedulingDecision.executionMode + executionModeReason = schedulingDecision.executionReason + } + executionRoute = routeDecision.route routeReason = routeDecision.reason @@ -114,6 +152,12 @@ export async function executeDesktopClickTarget( executionRoute = 'os_input' routeReason = `browser-dom extension transport does not support ${requiredActions.join(' + ')}` routeNote = `browser-dom ${routeDecision.bridgeMethod ?? 'click'} is unavailable on the connected extension transport (${getUnsupportedBrowserDomActions(runtime.browserDomBridge, ...requiredActions).join(', ')} unsupported), fell back to OS input` + executionMode = 'foreground' + executionModeReason = 'browser_dom is unavailable, so native input fallback needs foreground access' + await ensureForegroundForOsInput() + routeNote = routeNote + ? `${routeNote}; focused ${fallbackForegroundApp ?? 'target app'} before OS input fallback` + : `focused ${fallbackForegroundApp ?? 'target app'} before OS input fallback` osInputResult = await executeOsClick() } else { @@ -135,11 +179,18 @@ export async function executeDesktopClickTarget( catch (browserError) { executionRoute = 'os_input' routeNote = `browser-dom ${routeDecision.bridgeMethod ?? 'click'} failed (${errorMessageFrom(browserError) ?? 'unknown error'}), fell back to OS input` + executionMode = 'foreground' + executionModeReason = 'browser_dom failed, so native input fallback needs foreground access' + await ensureForegroundForOsInput() + routeNote = `${routeNote}; focused ${fallbackForegroundApp ?? 'target app'} before OS input fallback` osInputResult = await executeOsClick() } } } else { + executionMode = schedulingDecision.executionMode + executionModeReason = schedulingDecision.executionReason + await ensureForegroundForOsInput() osInputResult = await executeOsClick() } @@ -157,6 +208,7 @@ export async function executeDesktopClickTarget( ` Snap: ${snap.reason}`, ` Point: (${snap.snappedPoint.x}, ${snap.snappedPoint.y})`, ` Route: ${executionRoute} (${routeReason})`, + ` Execution mode: ${executionMode} (${executionModeReason})`, ` Button: ${button || 'left'}, clicks: ${clickCount ?? 1}`, ] @@ -177,6 +229,8 @@ export async function executeDesktopClickTarget( candidate, executionRoute, routeReason, + executionMode, + executionModeReason, routeNote: routeNote || undefined, osInputResult, }, diff --git a/services/computer-use-mcp/src/server/desktop-scheduler.test.ts b/services/computer-use-mcp/src/server/desktop-scheduler.test.ts new file mode 100644 index 000000000..ed1d2fb06 --- /dev/null +++ b/services/computer-use-mcp/src/server/desktop-scheduler.test.ts @@ -0,0 +1,83 @@ +import type { BrowserSurfaceAvailability } from '../types' + +import { describe, expect, it } from 'vitest' + +import { decideDesktopExecutionMode } from './desktop-scheduler' + +function makeBrowserSurfaceAvailability(): BrowserSurfaceAvailability { + return { + executionMode: 'local-windowed' as const, + suitable: true, + availableSurfaces: ['browser_dom', 'browser_cdp'], + preferredSurface: 'browser_dom' as const, + selectedToolName: 'browser_dom_read_page' as const, + reason: 'connected', + extension: { + enabled: true, + connected: true, + }, + cdp: { + endpoint: 'http://localhost:9222', + connected: true, + connectable: true, + }, + } +} + +describe('decideDesktopExecutionMode', () => { + it('keeps desktop_observe background when Chrome capture is disabled', () => { + const decision = decideDesktopExecutionMode({ + action: { kind: 'desktop_observe', input: { includeChrome: false } }, + }) + + expect(decision).toMatchObject({ + executionMode: 'background', + foregroundRequired: false, + }) + }) + + it('treats desktop_observe as browser_surface when browser surfaces are available', () => { + const decision = decideDesktopExecutionMode({ + action: { kind: 'desktop_observe', input: { includeChrome: true } }, + browserSurface: makeBrowserSurfaceAvailability(), + }) + + expect(decision).toMatchObject({ + executionMode: 'browser_surface', + browserSurfacePreferred: true, + foregroundRequired: false, + }) + }) + + it('keeps desktop_click_target background-safe when browser_dom is available', () => { + const decision = decideDesktopExecutionMode({ + action: { kind: 'desktop_click_target', input: { candidateId: 't_0' } }, + browserSurface: makeBrowserSurfaceAvailability(), + browserDomRoute: true, + }) + + expect(decision).toMatchObject({ + executionMode: 'browser_surface', + browserSurfacePreferred: true, + foregroundRequired: false, + }) + }) + + it('treats clipboard and wait actions as background-safe', () => { + const waitDecision = decideDesktopExecutionMode({ + action: { kind: 'wait', input: { durationMs: 250 } }, + }) + const clipboardDecision = decideDesktopExecutionMode({ + action: { kind: 'clipboard_read_text', input: {} }, + }) + + expect(waitDecision).toMatchObject({ + executionMode: 'background', + foregroundRequired: false, + }) + expect(clipboardDecision).toMatchObject({ + executionMode: 'background', + foregroundRequired: false, + }) + }) +}) diff --git a/services/computer-use-mcp/src/server/desktop-scheduler.ts b/services/computer-use-mcp/src/server/desktop-scheduler.ts new file mode 100644 index 000000000..ddd22f7f2 --- /dev/null +++ b/services/computer-use-mcp/src/server/desktop-scheduler.ts @@ -0,0 +1,118 @@ +import type { ActionInvocation, BrowserSurfaceAvailability } from '../types' + +export type DesktopExecutionMode = 'background' | 'browser_surface' | 'foreground' + +export interface DesktopSchedulingDecision { + executionMode: DesktopExecutionMode + executionReason: string + browserSurfacePreferred: boolean + foregroundRequired: boolean +} + +function hasBrowserDomSurface(browserSurface?: BrowserSurfaceAvailability): boolean { + return Boolean(browserSurface?.availableSurfaces?.some(surface => surface === 'browser_dom' || surface === 'browser_cdp')) +} + +function isBackgroundReadAction(action: ActionInvocation): boolean { + return action.kind === 'observe_windows' + || action.kind === 'screenshot' + || action.kind === 'wait' + || action.kind === 'clipboard_read_text' + || action.kind === 'clipboard_write_text' + || action.kind === 'secret_read_env_value' +} + +function isBrowserDomCapableAction(action: ActionInvocation): boolean { + return action.kind === 'desktop_click_target' +} + +function isNativeForegroundAction(action: ActionInvocation): boolean { + return action.kind === 'click' + || action.kind === 'press_keys' + || action.kind === 'scroll' + || action.kind === 'open_app' + || action.kind === 'focus_app' + || action.kind === 'terminal_exec' + || action.kind === 'terminal_reset' +} + +export function decideDesktopExecutionMode(params: { + action: ActionInvocation + browserSurface?: BrowserSurfaceAvailability + browserDomRoute?: boolean +}): DesktopSchedulingDecision { + const { action, browserSurface, browserDomRoute } = params + const browserSurfaceAvailable = hasBrowserDomSurface(browserSurface) + + if (action.kind === 'desktop_observe') { + if (action.input?.includeChrome === false) { + return { + executionMode: 'background', + executionReason: 'desktop_observe is background-only when Chrome capture is disabled', + browserSurfacePreferred: false, + foregroundRequired: false, + } + } + + if (browserSurfaceAvailable) { + return { + executionMode: 'browser_surface', + executionReason: 'browser surface is available, so desktop_observe can collect Chrome semantics without a foreground switch', + browserSurfacePreferred: true, + foregroundRequired: false, + } + } + + return { + executionMode: 'background', + executionReason: 'desktop_observe stays background-only because no browser surface is available', + browserSurfacePreferred: false, + foregroundRequired: false, + } + } + + if (isBackgroundReadAction(action)) { + return { + executionMode: 'background', + executionReason: `${action.kind} is read-only and does not need foreground switching`, + browserSurfacePreferred: false, + foregroundRequired: false, + } + } + + if (isBrowserDomCapableAction(action)) { + if (browserDomRoute) { + return { + executionMode: 'browser_surface', + executionReason: 'browser_dom route is available, so click_target can stay background-safe', + browserSurfacePreferred: true, + foregroundRequired: false, + } + } + + return { + executionMode: 'foreground', + executionReason: browserSurfaceAvailable + ? 'desktop_click_target needs foreground because browser_dom is unavailable for this candidate' + : 'desktop_click_target needs foreground because no browser surface is available', + browserSurfacePreferred: false, + foregroundRequired: true, + } + } + + if (isNativeForegroundAction(action)) { + return { + executionMode: 'foreground', + executionReason: `${action.kind} uses native input and needs foreground access`, + browserSurfacePreferred: false, + foregroundRequired: true, + } + } + + return { + executionMode: 'foreground', + executionReason: `defaulting ${action.kind} to foreground execution`, + browserSurfacePreferred: false, + foregroundRequired: true, + } +} diff --git a/services/computer-use-mcp/src/server/register-desktop-grounding-tools.test.ts b/services/computer-use-mcp/src/server/register-desktop-grounding-tools.test.ts index 883bf7fc3..658ecc188 100644 --- a/services/computer-use-mcp/src/server/register-desktop-grounding-tools.test.ts +++ b/services/computer-use-mcp/src/server/register-desktop-grounding-tools.test.ts @@ -165,4 +165,31 @@ describe('registerDesktopGroundingTools', () => { }), ]) }) + + it('does not refocus before desktop_observe', async () => { + const { runtime, executeAction } = createRuntime() + const { server, invoke } = createMockServer() + registerDesktopGroundingTools({ server, runtime, executeAction }) + + captureDesktopGroundingMock.mockResolvedValueOnce({ + snapshotId: 'dg_bg', + capturedAt: new Date().toISOString(), + foregroundApp: 'Google Chrome', + windows: [], + screenshot: { + dataBase64: '', + mimeType: 'image/png', + path: '/tmp/shot.png', + capturedAt: new Date().toISOString(), + }, + targetCandidates: [], + staleFlags: { screenshot: false, ax: false, chromeSemantic: false }, + } as any) + + const result = await invoke('desktop_observe', { includeChrome: true }) + + expect(result.isError).not.toBe(true) + expect(runtime.desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled() + expect(captureDesktopGroundingMock).toHaveBeenCalledOnce() + }) }) diff --git a/services/computer-use-mcp/src/server/register-desktop-grounding.ts b/services/computer-use-mcp/src/server/register-desktop-grounding.ts index 6e11fe986..c7631be24 100644 --- a/services/computer-use-mcp/src/server/register-desktop-grounding.ts +++ b/services/computer-use-mcp/src/server/register-desktop-grounding.ts @@ -23,7 +23,6 @@ import process from 'node:process' import { z } from 'zod' import { captureDesktopGrounding, formatGroundingForAgent } from '../desktop-grounding' -import { sleep } from '../utils/sleep' import { textContent } from './content' import { registerToolWithDescriptor, requireDescriptor } from './tool-descriptors/register-helper' @@ -50,47 +49,11 @@ export function registerDesktopGroundingTools(params: { descriptor: requireDescriptor('desktop_observe'), schema: { - includeChrome: z.boolean().optional().describe('Whether to include Chrome semantic data. Default: auto-detect based on foreground app.'), + includeChrome: z.boolean().optional().describe('Whether to include Chrome semantic data. Default: best-effort when browser surfaces are available.'), }, handler: async ({ includeChrome }) => { try { - // If the agent has a desktop session with a controlled app, - // ensure that app is in the foreground before observing. - // Falls back to Chrome session check for backward compatibility. - const sessionCtrl = runtime.desktopSessionController - const activeSession = sessionCtrl.getSession() - if (activeSession?.controlledApp) { - const currentForeground = await runtime.executor.getForegroundContext() - const wasAlreadyInFront = await sessionCtrl.ensureControlledAppInForeground({ - currentForeground, - chromeSessionManager: runtime.chromeSessionManager, - activateApp: async (appName) => { - await runtime.executor.focusApp({ app: appName }) - }, - }) - if (!wasAlreadyInFront) { - await sleep(300) - } - } - else { - // Fallback: Chrome session without desktop session - const chromeSession = runtime.chromeSessionManager.getSessionInfo() - if (chromeSession) { - const currentForeground = await runtime.executor.getForegroundContext() - if (currentForeground.available && currentForeground.appName !== 'Google Chrome') { - if (currentForeground.appName) { - runtime.stateManager.savePreviousUserForeground(currentForeground.appName) - } - const activated = await runtime.chromeSessionManager.bringToFront() - if (!activated) { - throw new Error('Chrome session is unavailable; call desktop_ensure_chrome before observing Chrome.') - } - await sleep(300) - } - } - } - // Try to get or reconnect a CDP bridge. // NOTICE: `desktop_ensure_chrome` can launch Chrome before its DevTools // endpoint is fully ready. When observe runs later, reconnect from the