feat(computer-use-mcp): add background desktop scheduler (#1805)

services/computer-use-mcp/src/chrome-semantic-adapter.ts

@@ -3,7 +3,7 @@
  * and maps it to DesktopTargetCandidate format.
  *
  * Uses the extension bridge as primary source and CDP bridge as fallback.
- * Only active when Chrome is the foreground app.
+ * Best effort when browser surfaces are available.
  *
  * The adapter handles coordinate transformation from page-relative
  * (CSS viewport) coordinates to screen-absolute coordinates using

services/computer-use-mcp/src/desktop-grounding-capture.test.ts

@@ -0,0 +1,107 @@
+import type { AXSnapshot } from './accessibility/types'
+import type { DesktopExecutor, WindowObservation } from './types'
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { captureDesktopGrounding } from './desktop-grounding'
+
+const { captureAXTreeMock, captureChromeSemanticsMock } = vi.hoisted(() => ({
+  captureAXTreeMock: vi.fn(),
+  captureChromeSemanticsMock: vi.fn(),
+}))
+
+vi.mock('./accessibility', () => ({
+  captureAXTree: captureAXTreeMock,
+}))
+
+vi.mock('./chrome-semantic-adapter', async () => {
+  const actual = await vi.importActual<typeof import('./chrome-semantic-adapter')>('./chrome-semantic-adapter')
+  return {
+    ...actual,
+    captureChromeSemantics: captureChromeSemanticsMock,
+  }
+})
+
+function makeAxSnapshot(): AXSnapshot {
+  const root = {
+    uid: 'root',
+    role: 'AXApplication',
+    title: 'AIRI',
+    children: [],
+  }
+
+  return {
+    snapshotId: 'ax_1',
+    pid: 123,
+    appName: 'AIRI',
+    root,
+    uidToNode: new Map([['root', root]]),
+    capturedAt: new Date().toISOString(),
+    maxDepth: 1,
+    truncated: false,
+  } as AXSnapshot
+}
+
+describe('captureDesktopGrounding', () => {
+  beforeEach(() => {
+    captureAXTreeMock.mockReset()
+    captureChromeSemanticsMock.mockReset()
+  })
+
+  it('does not project Chrome semantics onto non-Chrome windows that only share the same title', async () => {
+    captureAXTreeMock.mockResolvedValue(makeAxSnapshot())
+    captureChromeSemanticsMock.mockResolvedValue({
+      pageUrl: 'https://example.com',
+      pageTitle: 'Shared Title',
+      interactiveElements: [
+        {
+          tag: 'button',
+          text: 'Submit',
+          rect: { x: 20, y: 20, w: 80, h: 30 },
+        },
+      ],
+      capturedAt: new Date().toISOString(),
+      source: 'extension',
+    })
+
+    const genericObservation: WindowObservation = {
+      frontmostAppName: 'AIRI',
+      windows: [
+        {
+          id: 'airi:1',
+          appName: 'AIRI',
+          title: 'Shared Title',
+          bounds: { x: 10, y: 20, width: 1200, height: 800 },
+        },
+      ],
+      observedAt: new Date().toISOString(),
+    }
+
+    const chromeObservation: WindowObservation = {
+      windows: [],
+      observedAt: new Date().toISOString(),
+    }
+
+    const observeWindows = vi.fn()
+      .mockResolvedValueOnce(genericObservation)
+      .mockResolvedValueOnce(chromeObservation)
+
+    const executor = {
+      takeScreenshot: vi.fn().mockResolvedValue({
+        dataBase64: '',
+        mimeType: 'image/png',
+        path: '',
+        capturedAt: new Date().toISOString(),
+      }),
+      observeWindows,
+    } as unknown as DesktopExecutor
+
+    const snapshot = await captureDesktopGrounding({
+      config: {} as never,
+      executor,
+      input: { includeChrome: true },
+    })
+
+    expect(snapshot.targetCandidates.some(candidate => candidate.source === 'chrome_dom')).toBe(false)
+  })
+})

services/computer-use-mcp/src/desktop-grounding-types.ts

@@ -147,7 +147,7 @@ export interface DesktopGroundingSnapshot {
   screenshot: ScreenshotArtifact
   /** macOS AX tree snapshot (if captured successfully) */
   axSnapshot?: AXSnapshot
-  /** Chrome semantic snapshot (only when Chrome is foreground) */
+  /** Chrome semantic snapshot (best effort when browser surfaces are available) */
   chromeSemanticSnapshot?: ChromeSemanticSnapshot
   /** Merged, deduplicated, ranked target candidates */
   targetCandidates: DesktopTargetCandidate[]

services/computer-use-mcp/src/desktop-grounding.test.ts

@@ -104,6 +104,7 @@ describe('buildTargetCandidates', () => {
     expect(candidates.length).toBe(2)
     expect(candidates[0].source).toBe('chrome_dom')
     expect(candidates[0].tag).toBe('button')
+    expect(candidates[0].appName).toBe('Google Chrome')
   })
 
   it('chrome + AX: deduplicates overlapping candidates', () => {
@@ -168,6 +169,22 @@ describe('buildTargetCandidates', () => {
     const candidates = buildTargetCandidates({ axSnapshot: ax, foregroundApp: 'Finder' })
     expect(candidates[0].interactable).toBe(false)
   })
+
+  it('keeps chrome_dom candidates attached to Google Chrome even when another app is foreground', () => {
+    const chrome = makeChromeSnapshot([
+      { tag: 'button', text: 'Submit', rect: { x: 10, y: 10, w: 80, h: 30 } },
+    ])
+
+    const candidates = buildTargetCandidates({
+      chromeSnapshot: chrome,
+      chromeWindowBounds: { x: 0, y: 0, width: 1920, height: 1080 },
+      foregroundApp: 'Finder',
+    })
+
+    expect(candidates).toHaveLength(1)
+    expect(candidates[0].source).toBe('chrome_dom')
+    expect(candidates[0].appName).toBe('Google Chrome')
+  })
 })
 
 describe('captureDesktopGrounding', () => {

services/computer-use-mcp/src/desktop-grounding.ts

@@ -3,8 +3,8 @@
  *
  * This is the main entry point for the `desktop_observe` tool.
  * It captures screenshot, window observation, AX tree, and Chrome semantics
- * in parallel, then merges everything into a single `DesktopGroundingSnapshot`
- * with deduplicated, ranked target candidates.
+ * in parallel when available, then merges everything into a single
+ * `DesktopGroundingSnapshot` with deduplicated, ranked target candidates.
  */
 
 import type { AXNode, AXSnapshot } from './accessibility/types'
@@ -36,21 +36,13 @@ import { boundsIoU } from './snap-resolver'
  */
 const STALENESS_THRESHOLD_MS = 2000
 
-/** Known Chrome-like browser app names (lowercase, no .app suffix) */
-const CHROME_APPS = new Set([
-  'google chrome',
-  'chrome',
-  'google chrome canary',
-  'chromium',
-])
-
 let nextSnapshotId = 1
 
 /**
  * Capture a unified desktop grounding snapshot.
  *
  * Runs screenshot, window observation, and AX tree capture in parallel.
- * If the foreground app is Chrome (and `includeChrome` is not false),
+ * If Chrome browser surfaces are available (and `includeChrome` is not false),
  * also captures Chrome semantic data.
  *
  * @param params - Capture parameters (config, executor, input, bridges)
@@ -79,7 +71,27 @@ export async function captureDesktopGrounding(params: {
 
   // Determine foreground app
   const foregroundApp = windowObs.frontmostAppName || axSnapshot?.appName || 'unknown'
-  const isChromeInFront = isChromeApp(foregroundApp)
+  const shouldCaptureChrome = input?.includeChrome !== false
+
+  // Ask the executor for a Chrome-filtered window list when Chrome semantics
+  // are requested. The generic top-N window snapshot is often dominated by
+  // system UI and can miss Chrome entirely, which would prevent chrome_dom
+  // candidates from being mapped to screen coordinates.
+  let chromeWindowBounds = findChromeWindowBounds(windowObs)
+  let chromeWindowObservation: WindowObservation | undefined
+  if (shouldCaptureChrome && !chromeWindowBounds) {
+    try {
+      chromeWindowObservation = await executor.observeWindows({
+        app: 'Google Chrome',
+        limit: 12,
+      })
+      chromeWindowBounds = findChromeWindowBounds(chromeWindowObservation)
+    }
+    catch {
+      // Best-effort only. Fall back to AX-only candidates if filtered window
+      // enumeration fails.
+    }
+  }
 
   // If Chrome is foreground, ask the executor for a Chrome-filtered window list.
   // The generic top-N window snapshot is often dominated by system UI and can
@@ -102,8 +114,14 @@ export async function captureDesktopGrounding(params: {
 
   // Phase 2: Chrome semantic data (only if Chrome is foreground and allowed)
   let chromeSemanticSnapshot: ChromeSemanticSnapshot | null = null
-  if (isChromeInFront && input?.includeChrome !== false) {
+  if (shouldCaptureChrome) {
     chromeSemanticSnapshot = await captureChromeSemantics(extensionBridge, cdpBridge)
+    if (!chromeWindowBounds && chromeSemanticSnapshot?.pageTitle) {
+      chromeWindowBounds = findChromeWindowBounds(
+        chromeWindowObservation ?? windowObs,
+        chromeSemanticSnapshot.pageTitle,
+      )
+    }
   }
 
   // Phase 3: Build target candidates
@@ -120,7 +138,7 @@ export async function captureDesktopGrounding(params: {
     screenshot,
     axSnapshot,
     chromeSemanticSnapshot: chromeSemanticSnapshot ?? undefined,
-    isChromeInFront,
+    chromeSemanticEnabled: shouldCaptureChrome,
     assemblyTimestamp: now,
   })
 
@@ -162,10 +180,6 @@ export function buildTargetCandidates(params: {
       chromeSnapshot.interactiveElements,
       chromeWindowBounds,
     )
-    // Set appName on all chrome candidates
-    for (const c of chromeCandidates) {
-      c.appName = foregroundApp
-    }
   }
 
   // 2. Build AX candidates
@@ -329,17 +343,32 @@ function axNodesToTargetCandidates(
 // Helpers
 // ---------------------------------------------------------------------------
 
-/** Compiled regex for stripping .app suffix from macOS app names */
-const APP_SUFFIX_RE = /\.app$/u
-
-function isChromeApp(appName: string): boolean {
-  return CHROME_APPS.has(appName.trim().toLowerCase().replace(APP_SUFFIX_RE, ''))
-}
-
 function findChromeWindowBounds(
   observation: WindowObservation,
-  _foregroundApp: string,
+  titleHint?: string,
 ): Bounds | undefined {
+  const normalizedTitleHint = titleHint?.trim().toLowerCase()
+  if (normalizedTitleHint) {
+    const titleMatchedWindow = observation.windows.find((window) => {
+      if (!window.bounds || !window.appName.toLowerCase().includes('chrome')) {
+        return false
+      }
+
+      const normalizedWindowTitle = window.title?.trim().toLowerCase()
+      if (!normalizedWindowTitle) {
+        return false
+      }
+
+      return normalizedWindowTitle === normalizedTitleHint
+        || normalizedWindowTitle.includes(normalizedTitleHint)
+        || normalizedTitleHint.includes(normalizedWindowTitle)
+    })
+
+    if (titleMatchedWindow?.bounds) {
+      return titleMatchedWindow.bounds
+    }
+  }
+
   const chromeWindow = observation.windows.find(w =>
     w.appName.toLowerCase().includes('chrome') && w.bounds,
   )
@@ -350,10 +379,10 @@ function computeStaleness(params: {
   screenshot: ScreenshotArtifact
   axSnapshot?: AXSnapshot
   chromeSemanticSnapshot?: ChromeSemanticSnapshot
-  isChromeInFront: boolean
+  chromeSemanticEnabled: boolean
   assemblyTimestamp: number
 }): GroundingStalenessFlags {
-  const { screenshot, axSnapshot, chromeSemanticSnapshot, isChromeInFront, assemblyTimestamp } = params
+  const { screenshot, axSnapshot, chromeSemanticSnapshot, chromeSemanticEnabled, assemblyTimestamp } = params
 
   const screenshotStale = !screenshot.capturedAt
     || (assemblyTimestamp - new Date(screenshot.capturedAt).getTime()) > STALENESS_THRESHOLD_MS
@@ -363,7 +392,7 @@ function computeStaleness(params: {
     || !axSnapshot.capturedAt
     || (assemblyTimestamp - new Date(axSnapshot.capturedAt).getTime()) > STALENESS_THRESHOLD_MS
 
-  const chromeStale = !isChromeInFront
+  const chromeStale = !chromeSemanticEnabled
     || !chromeSemanticSnapshot
     || !chromeSemanticSnapshot.capturedAt
     || (assemblyTimestamp - new Date(chromeSemanticSnapshot.capturedAt).getTime()) > STALENESS_THRESHOLD_MS

services/computer-use-mcp/src/server/action-executor.test.ts

@@ -71,6 +71,9 @@ function createRuntimeForActionTest(configOverrides: Partial<ComputerUseConfig>
       connected: true,
       pendingRequests: 0,
     }),
+    supportsAction: vi.fn().mockReturnValue(true),
+    clickSelector: vi.fn().mockResolvedValue(undefined),
+    checkCheckbox: vi.fn().mockResolvedValue(undefined),
   }
   const cdpBridgeManager = {
     probeAvailability: vi.fn().mockResolvedValue({
@@ -323,6 +326,218 @@ describe('createExecuteAction', () => {
     }))
   })
 
+  it('does not refocus when desktop_click_target stays on browser_dom', async () => {
+    const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest()
+    stateManager.updateGroundingSnapshot({
+      snapshotId: 'dg_1',
+      capturedAt: new Date().toISOString(),
+      foregroundApp: 'Google Chrome',
+      windows: [],
+      screenshot: {
+        dataBase64: '',
+        mimeType: 'image/png',
+        path: '',
+        capturedAt: new Date().toISOString(),
+      },
+      targetCandidates: [
+        {
+          id: 't_0',
+          source: 'chrome_dom',
+          appName: 'Google Chrome',
+          role: 'button',
+          label: 'Submit',
+          bounds: { x: 100, y: 200, width: 80, height: 30 },
+          confidence: 0.95,
+          interactable: true,
+          selector: '#submit',
+          frameId: 0,
+          isPageContent: true,
+        },
+      ],
+      staleFlags: { screenshot: false, ax: false, chromeSemantic: false },
+    } as any)
+    desktopSessionController.getSession.mockReturnValue({
+      id: 'ds_1',
+      controlledApp: 'Google Chrome',
+      ownedWindows: [],
+      createdAt: new Date().toISOString(),
+      lastActiveAt: new Date().toISOString(),
+    })
+    executor.getForegroundContext.mockResolvedValue({
+      available: true,
+      appName: 'AIRI',
+      platform: 'darwin',
+    })
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target')
+
+    expect(result.isError).not.toBe(true)
+    expect(desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled()
+    expect(executor.focusApp).not.toHaveBeenCalled()
+    expect(executor.click).not.toHaveBeenCalled()
+    expect(result.structuredContent).toMatchObject({
+      status: 'executed',
+      backendResult: expect.objectContaining({
+        executionMode: 'browser_surface',
+      }),
+    })
+  })
+
+  it('refocuses when desktop_click_target falls back to OS input', async () => {
+    const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest()
+    stateManager.updateGroundingSnapshot({
+      snapshotId: 'dg_1',
+      capturedAt: new Date().toISOString(),
+      foregroundApp: 'Google Chrome',
+      windows: [],
+      screenshot: {
+        dataBase64: '',
+        mimeType: 'image/png',
+        path: '',
+        capturedAt: new Date().toISOString(),
+      },
+      targetCandidates: [
+        {
+          id: 't_0',
+          source: 'ax',
+          appName: 'Google Chrome',
+          role: 'AXButton',
+          label: 'Submit',
+          bounds: { x: 100, y: 200, width: 80, height: 30 },
+          confidence: 0.95,
+          interactable: true,
+        },
+      ],
+      staleFlags: { screenshot: false, ax: false, chromeSemantic: false },
+    } as any)
+    desktopSessionController.getSession.mockReturnValue({
+      id: 'ds_1',
+      controlledApp: 'Google Chrome',
+      ownedWindows: [],
+      createdAt: new Date().toISOString(),
+      lastActiveAt: new Date().toISOString(),
+    })
+    desktopSessionController.ensureControlledAppInForeground.mockResolvedValue(true)
+    executor.getForegroundContext.mockResolvedValue({
+      available: true,
+      appName: 'AIRI',
+      platform: 'darwin',
+    })
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target')
+
+    expect(result.isError).not.toBe(true)
+    expect(desktopSessionController.ensureControlledAppInForeground).toHaveBeenCalled()
+    expect(executor.click).toHaveBeenCalledOnce()
+  })
+
+  it('focuses the candidate app before OS fallback when no controlled session exists', async () => {
+    const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest()
+    stateManager.updateGroundingSnapshot({
+      snapshotId: 'dg_1',
+      capturedAt: new Date().toISOString(),
+      foregroundApp: 'Google Chrome',
+      windows: [],
+      screenshot: {
+        dataBase64: '',
+        mimeType: 'image/png',
+        path: '',
+        capturedAt: new Date().toISOString(),
+      },
+      targetCandidates: [
+        {
+          id: 't_0',
+          source: 'chrome_dom',
+          appName: 'Google Chrome',
+          role: 'button',
+          label: 'Submit',
+          bounds: { x: 100, y: 200, width: 80, height: 30 },
+          confidence: 0.95,
+          interactable: true,
+          selector: '#submit',
+          frameId: 0,
+          isPageContent: true,
+        },
+      ],
+      staleFlags: { screenshot: false, ax: false, chromeSemantic: false },
+    } as any)
+    desktopSessionController.getSession.mockReturnValue(null)
+    executor.getForegroundContext.mockResolvedValue({
+      available: true,
+      appName: 'Finder',
+      platform: 'darwin',
+    })
+    ;(runtime.browserDomBridge.supportsAction as any).mockReturnValue(false)
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target')
+
+    expect(result.isError).not.toBe(true)
+    expect(desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled()
+    expect(executor.focusApp).toHaveBeenCalledWith({ app: 'Google Chrome' })
+    expect(executor.click).toHaveBeenCalledOnce()
+    expect(result.structuredContent).toMatchObject({
+      status: 'executed',
+      backendResult: expect.objectContaining({
+        executionRoute: 'os_input',
+        executionMode: 'foreground',
+        routeNote: expect.stringContaining('focused Google Chrome before OS input fallback'),
+      }),
+    })
+  })
+
+  it('rejects cross-app desktop_click_target fallback under a controlled session', async () => {
+    const { runtime, executor, stateManager, desktopSessionController } = createRuntimeForActionTest()
+    stateManager.updateGroundingSnapshot({
+      snapshotId: 'dg_1',
+      capturedAt: new Date().toISOString(),
+      foregroundApp: 'AIRI',
+      windows: [],
+      screenshot: {
+        dataBase64: '',
+        mimeType: 'image/png',
+        path: '',
+        capturedAt: new Date().toISOString(),
+      },
+      targetCandidates: [
+        {
+          id: 't_0',
+          source: 'ax',
+          appName: 'AIRI',
+          role: 'AXButton',
+          label: 'Submit',
+          bounds: { x: 100, y: 200, width: 80, height: 30 },
+          confidence: 0.95,
+          interactable: true,
+        },
+      ],
+      staleFlags: { screenshot: false, ax: false, chromeSemantic: false },
+    } as any)
+    desktopSessionController.getSession.mockReturnValue({
+      id: 'ds_1',
+      controlledApp: 'Google Chrome',
+      ownedWindows: [],
+      createdAt: new Date().toISOString(),
+      lastActiveAt: new Date().toISOString(),
+    })
+    executor.getForegroundContext.mockResolvedValue({
+      available: true,
+      appName: 'AIRI',
+      platform: 'darwin',
+    })
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'desktop_click_target', input: { candidateId: 't_0' } }, 'desktop_click_target')
+
+    expect(result.isError).toBe(true)
+    expect(result.content.find(item => item.type === 'text')?.text).toContain('cross-app fallback')
+    expect(desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled()
+    expect(executor.focusApp).not.toHaveBeenCalled()
+    expect(executor.click).not.toHaveBeenCalled()
+  })
+
   it('returns a structured failure when controlled-app refocus fails during desktop_click_target execution', async () => {
     const { runtime, executor, session, stateManager, desktopSessionController } = createRuntimeForActionTest()
     stateManager.updateGroundingSnapshot({

services/computer-use-mcp/src/server/desktop-grounding-actions.ts

@@ -3,10 +3,12 @@ import type { ComputerUseServerRuntime } from './runtime'
 
 import { errorMessageFrom } from '@moeru/std'
 
+import { appNamesMatch } from '../app-aliases'
 import { decideBrowserAction } from '../browser-action-router'
 import { getUnsupportedBrowserDomActions, isBrowserDomActionSupported } from '../browser-dom/capabilities'
 import { resolveSnapByCandidate } from '../snap-resolver'
 import { sleep } from '../utils/sleep'
+import { decideDesktopExecutionMode } from './desktop-scheduler'
 
 const DESKTOP_CLICK_SNAPSHOT_MAX_AGE_MS = 5000
 
@@ -46,23 +48,6 @@ export async function executeDesktopClickTarget(
     throw new Error(`Candidate "${candidateId}" not found in snapshot "${snapshot.snapshotId}". Available candidates: ${snapshot.targetCandidates.map(c => c.id).join(', ')}`)
   }
 
-  const sessionCtrl = runtime.desktopSessionController
-  const activeSession = sessionCtrl.getSession()
-  if (activeSession?.controlledApp) {
-    const currentForeground = await runtime.executor.getForegroundContext()
-    const wasAlreadyInFront = await sessionCtrl.ensureControlledAppInForeground({
-      currentForeground,
-      chromeSessionManager: runtime.chromeSessionManager,
-      activateApp: async (appName) => {
-        await runtime.executor.focusApp({ app: appName })
-      },
-    })
-    if (!wasAlreadyInFront) {
-      await sleep(200)
-    }
-    sessionCtrl.touch()
-  }
-
   const candidate = snapshot.targetCandidates.find(c => c.id === candidateId)
   const intent = {
     mode: 'execute' as const,
@@ -83,6 +68,9 @@ export async function executeDesktopClickTarget(
   let routeNote = ''
   let routeReason = 'candidate not found'
   let osInputResult: ExecutorActionResult | undefined
+  let executionMode: 'background' | 'browser_surface' | 'foreground' = 'foreground'
+  let executionModeReason = 'desktop_click_target uses native input and needs foreground access'
+  let fallbackForegroundApp: string | undefined
 
   const executeOsClick = async () => {
     const result = await runtime.executor.click({
@@ -96,12 +84,62 @@ export async function executeDesktopClickTarget(
     return result
   }
 
+  const ensureForegroundForOsInput = async () => {
+    const sessionCtrl = runtime.desktopSessionController
+    const activeSession = sessionCtrl.getSession()
+    if (!activeSession?.controlledApp) {
+      const candidateApp = candidate?.appName?.trim()
+      if (!candidateApp || candidateApp === 'unknown') {
+        throw new Error('desktop_click_target cannot fall back to OS input without a controlled app session or a concrete target app.')
+      }
+
+      const currentForeground = await runtime.executor.getForegroundContext()
+      if (currentForeground.available && currentForeground.appName === candidateApp) {
+        fallbackForegroundApp = candidateApp
+        return
+      }
+
+      await runtime.executor.focusApp({ app: candidateApp })
+      fallbackForegroundApp = candidateApp
+      await sleep(200)
+      return
+    }
+
+    if (!appNamesMatch(candidate?.appName, activeSession.controlledApp)) {
+      throw new Error(`desktop_click_target rejected cross-app fallback: candidate "${candidate?.appName ?? 'unknown'}" does not match controlled app "${activeSession.controlledApp}".`)
+    }
+
+    const currentForeground = await runtime.executor.getForegroundContext()
+    const wasAlreadyInFront = await sessionCtrl.ensureControlledAppInForeground({
+      currentForeground,
+      chromeSessionManager: runtime.chromeSessionManager,
+      activateApp: async (appName) => {
+        await runtime.executor.focusApp({ app: appName })
+      },
+    })
+    fallbackForegroundApp = activeSession.controlledApp
+    if (!wasAlreadyInFront) {
+      await sleep(200)
+    }
+    sessionCtrl.touch()
+  }
+
   try {
     const bridgeConnected = runtime.browserDomBridge?.getStatus().connected ?? false
     const routeDecision = candidate
       ? decideBrowserAction(candidate, bridgeConnected, button, clickCount)
       : { route: 'os_input' as const, reason: 'candidate not found' }
 
+    const schedulingDecision = decideDesktopExecutionMode({
+      action: { kind: 'desktop_click_target', input },
+      browserSurface: runtime.stateManager.getState().browserSurfaceAvailability,
+      browserDomRoute: routeDecision.route === 'browser_dom',
+    })
+    if (routeDecision.route === 'browser_dom') {
+      executionMode = schedulingDecision.executionMode
+      executionModeReason = schedulingDecision.executionReason
+    }
+
     executionRoute = routeDecision.route
     routeReason = routeDecision.reason
 
@@ -114,6 +152,12 @@ export async function executeDesktopClickTarget(
         executionRoute = 'os_input'
         routeReason = `browser-dom extension transport does not support ${requiredActions.join(' + ')}`
         routeNote = `browser-dom ${routeDecision.bridgeMethod ?? 'click'} is unavailable on the connected extension transport (${getUnsupportedBrowserDomActions(runtime.browserDomBridge, ...requiredActions).join(', ')} unsupported), fell back to OS input`
+        executionMode = 'foreground'
+        executionModeReason = 'browser_dom is unavailable, so native input fallback needs foreground access'
+        await ensureForegroundForOsInput()
+        routeNote = routeNote
+          ? `${routeNote}; focused ${fallbackForegroundApp ?? 'target app'} before OS input fallback`
+          : `focused ${fallbackForegroundApp ?? 'target app'} before OS input fallback`
         osInputResult = await executeOsClick()
       }
       else {
@@ -135,11 +179,18 @@ export async function executeDesktopClickTarget(
         catch (browserError) {
           executionRoute = 'os_input'
           routeNote = `browser-dom ${routeDecision.bridgeMethod ?? 'click'} failed (${errorMessageFrom(browserError) ?? 'unknown error'}), fell back to OS input`
+          executionMode = 'foreground'
+          executionModeReason = 'browser_dom failed, so native input fallback needs foreground access'
+          await ensureForegroundForOsInput()
+          routeNote = `${routeNote}; focused ${fallbackForegroundApp ?? 'target app'} before OS input fallback`
           osInputResult = await executeOsClick()
         }
       }
     }
     else {
+      executionMode = schedulingDecision.executionMode
+      executionModeReason = schedulingDecision.executionReason
+      await ensureForegroundForOsInput()
       osInputResult = await executeOsClick()
     }
 
@@ -157,6 +208,7 @@ export async function executeDesktopClickTarget(
       `  Snap: ${snap.reason}`,
       `  Point: (${snap.snappedPoint.x}, ${snap.snappedPoint.y})`,
       `  Route: ${executionRoute} (${routeReason})`,
+      `  Execution mode: ${executionMode} (${executionModeReason})`,
       `  Button: ${button || 'left'}, clicks: ${clickCount ?? 1}`,
     ]
 
@@ -177,6 +229,8 @@ export async function executeDesktopClickTarget(
         candidate,
         executionRoute,
         routeReason,
+        executionMode,
+        executionModeReason,
         routeNote: routeNote || undefined,
         osInputResult,
       },

services/computer-use-mcp/src/server/desktop-scheduler.test.ts

@@ -0,0 +1,83 @@
+import type { BrowserSurfaceAvailability } from '../types'
+
+import { describe, expect, it } from 'vitest'
+
+import { decideDesktopExecutionMode } from './desktop-scheduler'
+
+function makeBrowserSurfaceAvailability(): BrowserSurfaceAvailability {
+  return {
+    executionMode: 'local-windowed' as const,
+    suitable: true,
+    availableSurfaces: ['browser_dom', 'browser_cdp'],
+    preferredSurface: 'browser_dom' as const,
+    selectedToolName: 'browser_dom_read_page' as const,
+    reason: 'connected',
+    extension: {
+      enabled: true,
+      connected: true,
+    },
+    cdp: {
+      endpoint: 'http://localhost:9222',
+      connected: true,
+      connectable: true,
+    },
+  }
+}
+
+describe('decideDesktopExecutionMode', () => {
+  it('keeps desktop_observe background when Chrome capture is disabled', () => {
+    const decision = decideDesktopExecutionMode({
+      action: { kind: 'desktop_observe', input: { includeChrome: false } },
+    })
+
+    expect(decision).toMatchObject({
+      executionMode: 'background',
+      foregroundRequired: false,
+    })
+  })
+
+  it('treats desktop_observe as browser_surface when browser surfaces are available', () => {
+    const decision = decideDesktopExecutionMode({
+      action: { kind: 'desktop_observe', input: { includeChrome: true } },
+      browserSurface: makeBrowserSurfaceAvailability(),
+    })
+
+    expect(decision).toMatchObject({
+      executionMode: 'browser_surface',
+      browserSurfacePreferred: true,
+      foregroundRequired: false,
+    })
+  })
+
+  it('keeps desktop_click_target background-safe when browser_dom is available', () => {
+    const decision = decideDesktopExecutionMode({
+      action: { kind: 'desktop_click_target', input: { candidateId: 't_0' } },
+      browserSurface: makeBrowserSurfaceAvailability(),
+      browserDomRoute: true,
+    })
+
+    expect(decision).toMatchObject({
+      executionMode: 'browser_surface',
+      browserSurfacePreferred: true,
+      foregroundRequired: false,
+    })
+  })
+
+  it('treats clipboard and wait actions as background-safe', () => {
+    const waitDecision = decideDesktopExecutionMode({
+      action: { kind: 'wait', input: { durationMs: 250 } },
+    })
+    const clipboardDecision = decideDesktopExecutionMode({
+      action: { kind: 'clipboard_read_text', input: {} },
+    })
+
+    expect(waitDecision).toMatchObject({
+      executionMode: 'background',
+      foregroundRequired: false,
+    })
+    expect(clipboardDecision).toMatchObject({
+      executionMode: 'background',
+      foregroundRequired: false,
+    })
+  })
+})

services/computer-use-mcp/src/server/desktop-scheduler.ts

@@ -0,0 +1,118 @@
+import type { ActionInvocation, BrowserSurfaceAvailability } from '../types'
+
+export type DesktopExecutionMode = 'background' | 'browser_surface' | 'foreground'
+
+export interface DesktopSchedulingDecision {
+  executionMode: DesktopExecutionMode
+  executionReason: string
+  browserSurfacePreferred: boolean
+  foregroundRequired: boolean
+}
+
+function hasBrowserDomSurface(browserSurface?: BrowserSurfaceAvailability): boolean {
+  return Boolean(browserSurface?.availableSurfaces?.some(surface => surface === 'browser_dom' || surface === 'browser_cdp'))
+}
+
+function isBackgroundReadAction(action: ActionInvocation): boolean {
+  return action.kind === 'observe_windows'
+    || action.kind === 'screenshot'
+    || action.kind === 'wait'
+    || action.kind === 'clipboard_read_text'
+    || action.kind === 'clipboard_write_text'
+    || action.kind === 'secret_read_env_value'
+}
+
+function isBrowserDomCapableAction(action: ActionInvocation): boolean {
+  return action.kind === 'desktop_click_target'
+}
+
+function isNativeForegroundAction(action: ActionInvocation): boolean {
+  return action.kind === 'click'
+    || action.kind === 'press_keys'
+    || action.kind === 'scroll'
+    || action.kind === 'open_app'
+    || action.kind === 'focus_app'
+    || action.kind === 'terminal_exec'
+    || action.kind === 'terminal_reset'
+}
+
+export function decideDesktopExecutionMode(params: {
+  action: ActionInvocation
+  browserSurface?: BrowserSurfaceAvailability
+  browserDomRoute?: boolean
+}): DesktopSchedulingDecision {
+  const { action, browserSurface, browserDomRoute } = params
+  const browserSurfaceAvailable = hasBrowserDomSurface(browserSurface)
+
+  if (action.kind === 'desktop_observe') {
+    if (action.input?.includeChrome === false) {
+      return {
+        executionMode: 'background',
+        executionReason: 'desktop_observe is background-only when Chrome capture is disabled',
+        browserSurfacePreferred: false,
+        foregroundRequired: false,
+      }
+    }
+
+    if (browserSurfaceAvailable) {
+      return {
+        executionMode: 'browser_surface',
+        executionReason: 'browser surface is available, so desktop_observe can collect Chrome semantics without a foreground switch',
+        browserSurfacePreferred: true,
+        foregroundRequired: false,
+      }
+    }
+
+    return {
+      executionMode: 'background',
+      executionReason: 'desktop_observe stays background-only because no browser surface is available',
+      browserSurfacePreferred: false,
+      foregroundRequired: false,
+    }
+  }
+
+  if (isBackgroundReadAction(action)) {
+    return {
+      executionMode: 'background',
+      executionReason: `${action.kind} is read-only and does not need foreground switching`,
+      browserSurfacePreferred: false,
+      foregroundRequired: false,
+    }
+  }
+
+  if (isBrowserDomCapableAction(action)) {
+    if (browserDomRoute) {
+      return {
+        executionMode: 'browser_surface',
+        executionReason: 'browser_dom route is available, so click_target can stay background-safe',
+        browserSurfacePreferred: true,
+        foregroundRequired: false,
+      }
+    }
+
+    return {
+      executionMode: 'foreground',
+      executionReason: browserSurfaceAvailable
+        ? 'desktop_click_target needs foreground because browser_dom is unavailable for this candidate'
+        : 'desktop_click_target needs foreground because no browser surface is available',
+      browserSurfacePreferred: false,
+      foregroundRequired: true,
+    }
+  }
+
+  if (isNativeForegroundAction(action)) {
+    return {
+      executionMode: 'foreground',
+      executionReason: `${action.kind} uses native input and needs foreground access`,
+      browserSurfacePreferred: false,
+      foregroundRequired: true,
+    }
+  }
+
+  return {
+    executionMode: 'foreground',
+    executionReason: `defaulting ${action.kind} to foreground execution`,
+    browserSurfacePreferred: false,
+    foregroundRequired: true,
+  }
+}

services/computer-use-mcp/src/server/register-desktop-grounding-tools.test.ts

@@ -165,4 +165,31 @@ describe('registerDesktopGroundingTools', () => {
       }),
     ])
   })
+
+  it('does not refocus before desktop_observe', async () => {
+    const { runtime, executeAction } = createRuntime()
+    const { server, invoke } = createMockServer()
+    registerDesktopGroundingTools({ server, runtime, executeAction })
+
+    captureDesktopGroundingMock.mockResolvedValueOnce({
+      snapshotId: 'dg_bg',
+      capturedAt: new Date().toISOString(),
+      foregroundApp: 'Google Chrome',
+      windows: [],
+      screenshot: {
+        dataBase64: '',
+        mimeType: 'image/png',
+        path: '/tmp/shot.png',
+        capturedAt: new Date().toISOString(),
+      },
+      targetCandidates: [],
+      staleFlags: { screenshot: false, ax: false, chromeSemantic: false },
+    } as any)
+
+    const result = await invoke('desktop_observe', { includeChrome: true })
+
+    expect(result.isError).not.toBe(true)
+    expect(runtime.desktopSessionController.ensureControlledAppInForeground).not.toHaveBeenCalled()
+    expect(captureDesktopGroundingMock).toHaveBeenCalledOnce()
+  })
 })

services/computer-use-mcp/src/server/register-desktop-grounding.ts

@@ -23,7 +23,6 @@ import process from 'node:process'
 import { z } from 'zod'
 
 import { captureDesktopGrounding, formatGroundingForAgent } from '../desktop-grounding'
-import { sleep } from '../utils/sleep'
 import { textContent } from './content'
 import { registerToolWithDescriptor, requireDescriptor } from './tool-descriptors/register-helper'
 
@@ -50,47 +49,11 @@ export function registerDesktopGroundingTools(params: {
     descriptor: requireDescriptor('desktop_observe'),
 
     schema: {
-      includeChrome: z.boolean().optional().describe('Whether to include Chrome semantic data. Default: auto-detect based on foreground app.'),
+      includeChrome: z.boolean().optional().describe('Whether to include Chrome semantic data. Default: best-effort when browser surfaces are available.'),
     },
 
     handler: async ({ includeChrome }) => {
       try {
-        // If the agent has a desktop session with a controlled app,
-        // ensure that app is in the foreground before observing.
-        // Falls back to Chrome session check for backward compatibility.
-        const sessionCtrl = runtime.desktopSessionController
-        const activeSession = sessionCtrl.getSession()
-        if (activeSession?.controlledApp) {
-          const currentForeground = await runtime.executor.getForegroundContext()
-          const wasAlreadyInFront = await sessionCtrl.ensureControlledAppInForeground({
-            currentForeground,
-            chromeSessionManager: runtime.chromeSessionManager,
-            activateApp: async (appName) => {
-              await runtime.executor.focusApp({ app: appName })
-            },
-          })
-          if (!wasAlreadyInFront) {
-            await sleep(300)
-          }
-        }
-        else {
-          // Fallback: Chrome session without desktop session
-          const chromeSession = runtime.chromeSessionManager.getSessionInfo()
-          if (chromeSession) {
-            const currentForeground = await runtime.executor.getForegroundContext()
-            if (currentForeground.available && currentForeground.appName !== 'Google Chrome') {
-              if (currentForeground.appName) {
-                runtime.stateManager.savePreviousUserForeground(currentForeground.appName)
-              }
-              const activated = await runtime.chromeSessionManager.bringToFront()
-              if (!activated) {
-                throw new Error('Chrome session is unavailable; call desktop_ensure_chrome before observing Chrome.')
-              }
-              await sleep(300)
-            }
-          }
-        }
-
         // Try to get or reconnect a CDP bridge.
         // NOTICE: `desktop_ensure_chrome` can launch Chrome before its DevTools
         // endpoint is fully ready. When observe runs later, reconnect from the