fix(computer-use-mcp): add macOS display geometry contract (#1750)

---------

Co-authored-by-agent: Antigravity <antigravity@gemini.com>

services/computer-use-mcp/src/display/display.test.ts

@@ -2,34 +2,34 @@ import type { DisplayDescriptor, MultiDisplaySnapshot } from '../display/types'
 
 import { describe, expect, it } from 'vitest'
 
-import { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from '../display/types'
+import { findDisplayForPoint, resolveDisplayPoint, toBackingPixelCoord, toDisplayLocalCoord, toGlobalCoord } from '../display/types'
 
 function createTestSnapshot(): MultiDisplaySnapshot {
   const mainDisplay: DisplayDescriptor = {
     displayId: 1,
     isMain: true,
     isBuiltIn: true,
-    bounds: { x: 0, y: 0, width: 1440, height: 900 },
-    visibleBounds: { x: 0, y: 25, width: 1440, height: 875 },
+    bounds: { x: 0, y: 0, width: 1512, height: 982 },
+    visibleBounds: { x: 0, y: 65, width: 1512, height: 884 },
     scaleFactor: 2,
-    pixelWidth: 2880,
-    pixelHeight: 1800,
+    pixelWidth: 3024,
+    pixelHeight: 1964,
   }
 
   const externalDisplay: DisplayDescriptor = {
-    displayId: 2,
+    displayId: 3,
     isMain: false,
     isBuiltIn: false,
-    bounds: { x: 1440, y: 0, width: 2560, height: 1440 },
-    visibleBounds: { x: 1440, y: 25, width: 2560, height: 1415 },
+    bounds: { x: -222, y: -1080, width: 1920, height: 1080 },
+    visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 },
     scaleFactor: 1,
-    pixelWidth: 2560,
-    pixelHeight: 1440,
+    pixelWidth: 1920,
+    pixelHeight: 1080,
   }
 
   return {
     displays: [mainDisplay, externalDisplay],
-    combinedBounds: { x: 0, y: 0, width: 4000, height: 1440 },
+    combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
     capturedAt: '2025-01-01T00:00:00.000Z',
   }
 }
@@ -37,7 +37,7 @@ function createTestSnapshot(): MultiDisplaySnapshot {
 describe('findDisplayForPoint', () => {
   it('finds the main display for a point on it', () => {
     const snapshot = createTestSnapshot()
-    const display = findDisplayForPoint(snapshot, 720, 450)
+    const display = findDisplayForPoint(snapshot, 756, 491)
 
     expect(display).toBeDefined()
     expect(display!.displayId).toBe(1)
@@ -46,10 +46,10 @@ describe('findDisplayForPoint', () => {
 
   it('finds the external display for a point on it', () => {
     const snapshot = createTestSnapshot()
-    const display = findDisplayForPoint(snapshot, 2000, 500)
+    const display = findDisplayForPoint(snapshot, -100, -500)
 
     expect(display).toBeDefined()
-    expect(display!.displayId).toBe(2)
+    expect(display!.displayId).toBe(3)
     expect(display!.isMain).toBe(false)
   })
 
@@ -62,53 +62,53 @@ describe('findDisplayForPoint', () => {
 
   it('handles edge case at display boundary', () => {
     const snapshot = createTestSnapshot()
-    // x=1440 is the first pixel of the external display
-    const display = findDisplayForPoint(snapshot, 1440, 0)
+    // y=0 is the first row of the main display below the upper external display.
+    const display = findDisplayForPoint(snapshot, 0, 0)
 
     expect(display).toBeDefined()
-    expect(display!.displayId).toBe(2)
+    expect(display!.displayId).toBe(1)
   })
 })
 
 describe('toDisplayLocalCoord', () => {
   it('converts global to local coordinates on main display', () => {
     const snapshot = createTestSnapshot()
-    const local = toDisplayLocalCoord(snapshot.displays[0], 720, 450)
+    const local = toDisplayLocalCoord(snapshot.displays[0], 756, 491)
 
-    expect(local.x).toBe(720)
-    expect(local.y).toBe(450)
+    expect(local.x).toBe(756)
+    expect(local.y).toBe(491)
   })
 
   it('converts global to local coordinates on external display', () => {
     const snapshot = createTestSnapshot()
-    const local = toDisplayLocalCoord(snapshot.displays[1], 2000, 500)
+    const local = toDisplayLocalCoord(snapshot.displays[1], -100, -500)
 
-    expect(local.x).toBe(560) // 2000 - 1440
-    expect(local.y).toBe(500)
+    expect(local.x).toBe(122)
+    expect(local.y).toBe(580)
   })
 })
 
 describe('toGlobalCoord', () => {
   it('converts local back to global on main display', () => {
     const snapshot = createTestSnapshot()
-    const global = toGlobalCoord(snapshot.displays[0], 720, 450)
+    const global = toGlobalCoord(snapshot.displays[0], 756, 491)
 
-    expect(global.x).toBe(720)
-    expect(global.y).toBe(450)
+    expect(global.x).toBe(756)
+    expect(global.y).toBe(491)
   })
 
   it('converts local back to global on external display', () => {
     const snapshot = createTestSnapshot()
-    const global = toGlobalCoord(snapshot.displays[1], 560, 500)
+    const global = toGlobalCoord(snapshot.displays[1], 122, 580)
 
-    expect(global.x).toBe(2000)
-    expect(global.y).toBe(500)
+    expect(global.x).toBe(-100)
+    expect(global.y).toBe(-500)
   })
 
   it('is inverse of toDisplayLocalCoord', () => {
     const snapshot = createTestSnapshot()
     const display = snapshot.displays[1]
-    const globalPoint = { x: 2500, y: 800 }
+    const globalPoint = { x: 800, y: -100 }
     const local = toDisplayLocalCoord(display, globalPoint.x, globalPoint.y)
     const roundTrip = toGlobalCoord(display, local.x, local.y)
 
@@ -116,3 +116,43 @@ describe('toGlobalCoord', () => {
     expect(roundTrip.y).toBe(globalPoint.y)
   })
 })
+
+describe('toBackingPixelCoord', () => {
+  it('maps Retina display-local logical coordinates to backing pixels', () => {
+    const snapshot = createTestSnapshot()
+    const backing = toBackingPixelCoord(snapshot.displays[0], 100, 50)
+
+    expect(backing).toEqual({ x: 200, y: 100 })
+  })
+
+  it('leaves 1x external display-local coordinates unchanged', () => {
+    const snapshot = createTestSnapshot()
+    const backing = toBackingPixelCoord(snapshot.displays[1], 122, 580)
+
+    expect(backing).toEqual({ x: 122, y: 580 })
+  })
+})
+
+describe('resolveDisplayPoint', () => {
+  it('resolves main Retina global point without scaling the original global coordinate', () => {
+    const snapshot = createTestSnapshot()
+    const resolved = resolveDisplayPoint(snapshot, 100, 50)
+
+    expect(resolved).toBeDefined()
+    expect(resolved!.global).toEqual({ x: 100, y: 50 })
+    expect(resolved!.local).toEqual({ x: 100, y: 50 })
+    expect(resolved!.backingPixel).toEqual({ x: 200, y: 100 })
+    expect(resolved!.display.displayId).toBe(1)
+  })
+
+  it('resolves negative-coordinate external display points', () => {
+    const snapshot = createTestSnapshot()
+    const resolved = resolveDisplayPoint(snapshot, -100, -500)
+
+    expect(resolved).toBeDefined()
+    expect(resolved!.global).toEqual({ x: -100, y: -500 })
+    expect(resolved!.local).toEqual({ x: 122, y: 580 })
+    expect(resolved!.backingPixel).toEqual({ x: 122, y: 580 })
+    expect(resolved!.display.displayId).toBe(3)
+  })
+})

services/computer-use-mcp/src/display/index.ts

@@ -1,3 +1,13 @@
 export { enumerateDisplays, formatDisplaySummary } from './enumerate'
-export { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from './types'
-export type { DisplayDescriptor, MultiDisplaySnapshot } from './types'
+export {
+  findDisplayForPoint,
+  resolveDisplayPoint,
+  toBackingPixelCoord,
+  toDisplayLocalCoord,
+  toGlobalCoord,
+} from './types'
+export type {
+  DisplayDescriptor,
+  DisplayPointResolution,
+  MultiDisplaySnapshot,
+} from './types'

services/computer-use-mcp/src/display/types.ts

@@ -42,6 +42,33 @@ export interface MultiDisplaySnapshot {
   capturedAt: string
 }
 
+/**
+ * Resolved position metadata for a point in AIRI's desktop coordinate space.
+ *
+ * The `global` coordinate is the unscaled logical point that macOS input events
+ * should receive. `local` and `backingPixel` are diagnostics for display-aware
+ * rendering, overlays, and Retina mismatch debugging.
+ */
+export interface DisplayPointResolution {
+  /** Original point in global logical screen coordinates. */
+  global: {
+    x: number
+    y: number
+  }
+  /** Display containing the global logical point. */
+  display: DisplayDescriptor
+  /** Point relative to the containing display in logical coordinates. */
+  local: {
+    x: number
+    y: number
+  }
+  /** Point relative to the containing display in backing pixels. */
+  backingPixel: {
+    x: number
+    y: number
+  }
+}
+
 /**
  * Given a logical screen point, find which display it belongs to.
  */
@@ -56,6 +83,30 @@ export function findDisplayForPoint(
   })
 }
 
+/**
+ * Converts display-local logical coordinates to backing pixels.
+ *
+ * Use when:
+ * - Rendering display-local overlay diagnostics
+ * - Comparing logical desktop points with backing-pixel screenshots
+ *
+ * Expects:
+ * - `localX` and `localY` are already relative to `display.bounds`
+ *
+ * Returns:
+ * - Display-local backing-pixel coordinates rounded to integer pixels
+ */
+export function toBackingPixelCoord(
+  display: DisplayDescriptor,
+  localX: number,
+  localY: number,
+): { x: number, y: number } {
+  return {
+    x: Math.round(localX * display.scaleFactor),
+    y: Math.round(localY * display.scaleFactor),
+  }
+}
+
 /**
  * Convert a logical coordinate to the local coordinate space of a specific display.
  */
@@ -70,6 +121,41 @@ export function toDisplayLocalCoord(
   }
 }
 
+/**
+ * Resolves a global logical point against the display snapshot.
+ *
+ * Use when:
+ * - Mapping desktop mutation targets to a concrete macOS display
+ * - Recording display-local and backing-pixel diagnostics
+ *
+ * Expects:
+ * - `snapshot` uses AIRI's top-left global logical coordinate space
+ * - `x` and `y` are not pre-scaled by Retina backing factor
+ *
+ * Returns:
+ * - Display metadata when the point is inside a connected display
+ * - `undefined` when the point is outside all display bounds
+ */
+export function resolveDisplayPoint(
+  snapshot: MultiDisplaySnapshot,
+  x: number,
+  y: number,
+): DisplayPointResolution | undefined {
+  const display = findDisplayForPoint(snapshot, x, y)
+  if (!display) {
+    return undefined
+  }
+
+  const local = toDisplayLocalCoord(display, x, y)
+
+  return {
+    global: { x, y },
+    display,
+    local,
+    backingPixel: toBackingPixelCoord(display, local.x, local.y),
+  }
+}
+
 /**
  * Convert display-local coordinates back to global logical coordinates.
  */

services/computer-use-mcp/src/runtime-probes.test.ts

@@ -1,14 +1,44 @@
+import type { MultiDisplaySnapshot } from './display'
 import type { ComputerUseConfig } from './types'
 
 import { describe, expect, it } from 'vitest'
 
-import { buildCoordinateSpaceInfo } from './runtime-probes'
+import { buildCoordinateSpaceInfo, buildDisplayInfoFromSnapshot } from './runtime-probes'
 import { createTestConfig } from './test-fixtures'
 
 const baseConfig: ComputerUseConfig = createTestConfig({
   allowedBounds: { x: 0, y: 0, width: 1440, height: 900 },
 })
 
+function createMultiDisplaySnapshot(): MultiDisplaySnapshot {
+  return {
+    displays: [
+      {
+        displayId: 1,
+        isMain: true,
+        isBuiltIn: true,
+        bounds: { x: 0, y: 0, width: 1512, height: 982 },
+        visibleBounds: { x: 0, y: 65, width: 1512, height: 884 },
+        scaleFactor: 2,
+        pixelWidth: 3024,
+        pixelHeight: 1964,
+      },
+      {
+        displayId: 3,
+        isMain: false,
+        isBuiltIn: false,
+        bounds: { x: -222, y: -1080, width: 1920, height: 1080 },
+        visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 },
+        scaleFactor: 1,
+        pixelWidth: 1920,
+        pixelHeight: 1080,
+      },
+    ],
+    combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
+    capturedAt: '2026-04-27T00:00:00.000Z',
+  }
+}
+
 describe('buildCoordinateSpaceInfo', () => {
   it('requires a screenshot before real input', () => {
     const info = buildCoordinateSpaceInfo({
@@ -59,4 +89,39 @@ describe('buildCoordinateSpaceInfo', () => {
     expect(info.aligned).toBe(false)
     expect(info.reason).toContain('Retina')
   })
+
+  it('keeps allowed bounds valid when they sit inside combined multi-display bounds', () => {
+    const info = buildCoordinateSpaceInfo({
+      config: createTestConfig({
+        allowedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
+      }),
+      lastScreenshot: {
+        path: '/tmp/screenshot.png',
+        width: 1920,
+        height: 2062,
+        placeholder: false,
+      },
+      displayInfo: buildDisplayInfoFromSnapshot(createMultiDisplaySnapshot(), 'darwin'),
+    })
+
+    expect(info.readyForMutations).toBe(true)
+    expect(info.aligned).toBe(true)
+  })
+})
+
+describe('buildDisplayInfoFromSnapshot', () => {
+  it('preserves legacy main-display facts while exposing multi-display bounds', () => {
+    const info = buildDisplayInfoFromSnapshot(createMultiDisplaySnapshot(), 'darwin')
+
+    expect(info.available).toBe(true)
+    expect(info.logicalWidth).toBe(1512)
+    expect(info.logicalHeight).toBe(982)
+    expect(info.pixelWidth).toBe(3024)
+    expect(info.pixelHeight).toBe(1964)
+    expect(info.scaleFactor).toBe(2)
+    expect(info.isRetina).toBe(true)
+    expect(info.displayCount).toBe(2)
+    expect(info.displays?.[1]?.bounds).toEqual({ x: -222, y: -1080, width: 1920, height: 1080 })
+    expect(info.combinedBounds).toEqual({ x: -222, y: -1080, width: 1920, height: 2062 })
+  })
 })

services/computer-use-mcp/src/runtime-probes.ts

@@ -1,3 +1,4 @@
+import type { MultiDisplaySnapshot } from './display'
 import type {
   ComputerUseConfig,
   CoordinateSpaceInfo,
@@ -12,6 +13,7 @@ import { hostname } from 'node:os'
 import { basename } from 'node:path'
 import { argv, pid, platform, ppid, title } from 'node:process'
 
+import { enumerateDisplays } from './display'
 import { runProcess } from './utils/process'
 import { runSwiftScript } from './utils/swift'
 
@@ -43,6 +45,53 @@ export function resolveLaunchContext(config: ComputerUseConfig): LaunchContext {
   }
 }
 
+/**
+ * Builds the public runtime display facts from the native display snapshot.
+ *
+ * Use when:
+ * - Publishing display facts through desktop capabilities/state
+ * - Preserving legacy main-display fields while exposing multi-display bounds
+ *
+ * Expects:
+ * - `snapshot` uses AIRI's top-left global logical coordinate space
+ *
+ * Returns:
+ * - DisplayInfo with legacy main-display fields and complete display list
+ */
+export function buildDisplayInfoFromSnapshot(
+  snapshot: MultiDisplaySnapshot,
+  targetPlatform: NodeJS.Platform = platform,
+): DisplayInfo {
+  const mainDisplay = snapshot.displays.find(display => display.isMain) ?? snapshot.displays[0]
+
+  if (!mainDisplay) {
+    return {
+      available: false,
+      platform: targetPlatform,
+      displayCount: 0,
+      displays: [],
+      combinedBounds: snapshot.combinedBounds,
+      capturedAt: snapshot.capturedAt,
+      note: 'display enumeration returned no connected displays',
+    }
+  }
+
+  return {
+    available: true,
+    platform: targetPlatform,
+    logicalWidth: mainDisplay.bounds.width,
+    logicalHeight: mainDisplay.bounds.height,
+    pixelWidth: mainDisplay.pixelWidth,
+    pixelHeight: mainDisplay.pixelHeight,
+    scaleFactor: mainDisplay.scaleFactor,
+    isRetina: mainDisplay.scaleFactor > 1,
+    displayCount: snapshot.displays.length,
+    displays: snapshot.displays,
+    combinedBounds: snapshot.combinedBounds,
+    capturedAt: snapshot.capturedAt,
+  }
+}
+
 export async function probeDisplayInfo(config: ComputerUseConfig): Promise<DisplayInfo> {
   if (platform !== 'darwin') {
     return {
@@ -52,43 +101,8 @@ export async function probeDisplayInfo(config: ComputerUseConfig): Promise<Displ
     }
   }
 
-  const script = `
-import AppKit
-import Foundation
-
-guard let screen = NSScreen.main else {
-  print("{\\"available\\":false,\\"note\\":\\"NSScreen.main unavailable\\"}")
-  exit(0)
-}
-
-let frame = screen.frame
-let scale = screen.backingScaleFactor
-let payload: [String: Any] = [
-  "available": true,
-  "logicalWidth": Int(frame.width),
-  "logicalHeight": Int(frame.height),
-  "pixelWidth": Int(frame.width * scale),
-  "pixelHeight": Int(frame.height * scale),
-  "scaleFactor": scale,
-  "isRetina": scale > 1.0
-]
-
-let data = try JSONSerialization.data(withJSONObject: payload, options: [])
-print(String(data: data, encoding: .utf8)!)
-`
-
   try {
-    const { stdout } = await runSwiftScript({
-      swiftBinary: config.binaries.swift,
-      timeoutMs: config.timeoutMs,
-      source: script,
-    })
-    const parsed = JSON.parse(stdout.trim()) as Omit<DisplayInfo, 'platform'>
-
-    return {
-      platform,
-      ...parsed,
-    }
+    return buildDisplayInfoFromSnapshot(await enumerateDisplays(config), platform)
   }
   catch (error) {
     return {

services/computer-use-mcp/src/server/action-executor.test.ts

@@ -108,6 +108,49 @@ function createRuntimeForActionTest(configOverrides: Partial<ComputerUseConfig>
   }
 }
 
+function createMultiDisplayInfo() {
+  return createDisplayInfo({
+    platform: 'darwin',
+    logicalWidth: 1512,
+    logicalHeight: 982,
+    pixelWidth: 3024,
+    pixelHeight: 1964,
+    scaleFactor: 2,
+    isRetina: true,
+    displayCount: 2,
+    displays: [
+      {
+        displayId: 1,
+        isMain: true,
+        isBuiltIn: true,
+        bounds: { x: 0, y: 0, width: 1512, height: 982 },
+        visibleBounds: { x: 0, y: 65, width: 1512, height: 884 },
+        scaleFactor: 2,
+        pixelWidth: 3024,
+        pixelHeight: 1964,
+      },
+      {
+        displayId: 3,
+        isMain: false,
+        isBuiltIn: false,
+        bounds: { x: -222, y: -1080, width: 1920, height: 1080 },
+        visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 },
+        scaleFactor: 1,
+        pixelWidth: 1920,
+        pixelHeight: 1080,
+      },
+    ],
+    combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
+    capturedAt: '2026-04-27T00:00:00.000Z',
+  })
+}
+
+function createCombinedDisplayBoundsConfig() {
+  return {
+    allowedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
+  }
+}
+
 describe('createExecuteAction', () => {
   it('executes desktop_click_target through the shared policy and audit pipeline', async () => {
     const { runtime, executor, session, stateManager } = createRuntimeForActionTest()
@@ -651,4 +694,99 @@ describe('createExecuteAction', () => {
     expect(executor.typeText).toHaveBeenCalledOnce()
     expect(browserDomBridge.setInputValue).not.toHaveBeenCalled()
   })
+
+  it('records main-display metadata while preserving original global logical click coordinates', async () => {
+    const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig())
+    executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'click', input: { x: 100, y: 50, button: 'left', captureAfter: false } }, 'desktop_click')
+
+    expect(result.isError).not.toBe(true)
+    expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({
+      x: 100,
+      y: 50,
+      pointerTrace: expect.arrayContaining([
+        expect.objectContaining({ x: 100, y: 50 }),
+      ]),
+    }))
+
+    const structured = result.structuredContent as Record<string, any>
+    expect(structured.backendResult.displayPoint).toMatchObject({
+      coordinateSpace: 'global-logical',
+      global: { x: 100, y: 50 },
+      displayId: 1,
+      local: { x: 100, y: 50 },
+      backingPixel: { x: 200, y: 100 },
+      scaleFactor: 2,
+    })
+  })
+
+  it('accepts negative-coordinate external display clicks and records display-local metadata', async () => {
+    const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig())
+    executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'click', input: { x: -100, y: -500, captureAfter: false } }, 'desktop_click')
+
+    expect(result.isError).not.toBe(true)
+    expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({
+      x: -100,
+      y: -500,
+      pointerTrace: expect.arrayContaining([
+        expect.objectContaining({ x: -100, y: -500 }),
+      ]),
+    }))
+
+    const structured = result.structuredContent as Record<string, any>
+    expect(structured.backendResult.displayPoint).toMatchObject({
+      coordinateSpace: 'global-logical',
+      global: { x: -100, y: -500 },
+      displayId: 3,
+      local: { x: 122, y: 580 },
+      backingPixel: { x: 122, y: 580 },
+      scaleFactor: 1,
+    })
+  })
+
+  it('rejects physical-pixel-looking Retina coordinates outside the global logical display contract', async () => {
+    const { runtime, executor } = createRuntimeForActionTest({
+      allowedBounds: { x: -10_000, y: -10_000, width: 20_000, height: 20_000 },
+    })
+    executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'click', input: { x: 2000, y: 500, captureAfter: false } }, 'desktop_click')
+
+    expect(result.isError).toBe(true)
+    expect(executor.click).not.toHaveBeenCalled()
+    expect((result.content[0] as { text: string }).text).toContain('outside connected display bounds')
+  })
+
+  it('uses the same display resolver for type_text preparatory clicks', async () => {
+    const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig())
+    executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
+    executor.typeText.mockResolvedValue({
+      performed: true,
+      backend: 'dry-run' as const,
+      notes: [],
+    })
+
+    const executeAction = createExecuteAction(runtime)
+    const result = await executeAction({ kind: 'type_text', input: { text: 'hello', x: -100, y: -500, captureAfter: false } }, 'desktop_type_text')
+
+    expect(result.isError).not.toBe(true)
+    expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({
+      x: -100,
+      y: -500,
+    }))
+    expect(executor.typeText).toHaveBeenCalledTimes(1)
+
+    const structured = result.structuredContent as Record<string, any>
+    expect(structured.backendResult.focusDisplayPoint).toMatchObject({
+      displayId: 3,
+      local: { x: 122, y: 580 },
+      backingPixel: { x: 122, y: 580 },
+    })
+  })
 })

services/computer-use-mcp/src/server/action-executor.ts

@@ -1,9 +1,11 @@
 import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'
 
+import type { DisplayPointResolution, MultiDisplaySnapshot } from '../display'
 import type {
   ActionInvocation,
   ComputerUseConfig,
   DesktopExecutor,
+  DisplayInfo,
   ForegroundContext,
   PolicyDecision,
   ScreenshotArtifact,
@@ -15,6 +17,7 @@ import type { ComputerUseServerRuntime } from './runtime'
 import { normalizeConfiguredAppAction } from '../app-aliases'
 import { decideBrowserTypeAction } from '../browser-action-router'
 import { isBrowserDomActionSupported } from '../browser-dom/capabilities'
+import { resolveDisplayPoint } from '../display'
 import { evaluateActionPolicy } from '../policy'
 import { getRuntimePreflight } from '../preflight'
 import { buildCoordinateSpaceInfo } from '../runtime-probes'
@@ -119,6 +122,91 @@ function toTerminalStateContent(state: TerminalState) {
   }
 }
 
+function displaySnapshotFromDisplayInfo(displayInfo: DisplayInfo): MultiDisplaySnapshot | undefined {
+  if (!displayInfo.displays?.length) {
+    return undefined
+  }
+
+  return {
+    displays: displayInfo.displays.map(display => ({
+      displayId: display.displayId,
+      isMain: display.isMain,
+      isBuiltIn: display.isBuiltIn,
+      bounds: display.bounds,
+      visibleBounds: display.visibleBounds,
+      scaleFactor: display.scaleFactor,
+      pixelWidth: display.pixelWidth,
+      pixelHeight: display.pixelHeight,
+    })),
+    combinedBounds: displayInfo.combinedBounds ?? {
+      x: 0,
+      y: 0,
+      width: 0,
+      height: 0,
+    },
+    capturedAt: displayInfo.capturedAt ?? new Date(0).toISOString(),
+  }
+}
+
+function getCoordinateMutationTarget(action: ActionInvocation): { x: number, y: number } | undefined {
+  switch (action.kind) {
+    case 'click':
+      return { x: action.input.x, y: action.input.y }
+    case 'type_text':
+      if (typeof action.input.x === 'number' && typeof action.input.y === 'number') {
+        return { x: action.input.x, y: action.input.y }
+      }
+      return undefined
+    case 'scroll':
+      if (typeof action.input.x === 'number' && typeof action.input.y === 'number') {
+        return { x: action.input.x, y: action.input.y }
+      }
+      return undefined
+    default:
+      return undefined
+  }
+}
+
+function toStructuredDisplayPoint(resolution: DisplayPointResolution) {
+  return {
+    coordinateSpace: 'global-logical',
+    global: resolution.global,
+    displayId: resolution.display.displayId,
+    displayBounds: resolution.display.bounds,
+    local: resolution.local,
+    backingPixel: resolution.backingPixel,
+    scaleFactor: resolution.display.scaleFactor,
+  }
+}
+
+function resolveActionDisplayPoint(action: ActionInvocation, displayInfo: DisplayInfo) {
+  const target = getCoordinateMutationTarget(action)
+  const snapshot = displaySnapshotFromDisplayInfo(displayInfo)
+
+  if (!target || !snapshot) {
+    return undefined
+  }
+
+  const resolution = resolveDisplayPoint(snapshot, target.x, target.y)
+  if (!resolution) {
+    const combined = displayInfo.combinedBounds
+    return {
+      status: 'outside' as const,
+      target,
+      reason: combined
+        ? `target point (${target.x}, ${target.y}) is outside connected display bounds ${combined.width}x${combined.height} @ (${combined.x},${combined.y})`
+        : `target point (${target.x}, ${target.y}) is outside connected display bounds`,
+    }
+  }
+
+  return {
+    status: 'ok' as const,
+    target,
+    resolution,
+    structured: toStructuredDisplayPoint(resolution),
+  }
+}
+
 function getPolicyEvaluationContext(params: {
   action: ActionInvocation
   actualContext: ForegroundContext
@@ -255,6 +343,33 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
       return buildDeniedResponse(decision, context, executionTarget)
     }
 
+    const actionDisplayPoint = resolveActionDisplayPoint(normalizedAction, displayInfo)
+    if (actionDisplayPoint?.status === 'outside') {
+      const deniedDecision = buildDeniedDecision({
+        decision,
+        issues: [actionDisplayPoint.reason],
+      })
+
+      await runtime.session.record({
+        event: 'denied',
+        toolName,
+        action: normalizedAction,
+        context,
+        policy: deniedDecision,
+        result: {
+          executionTarget,
+          displayInfo,
+          coordinateSpace: preflight.coordinateSpace,
+          targetPoint: actionDisplayPoint.target,
+        },
+      })
+
+      return buildDeniedResponse(deniedDecision, context, executionTarget)
+    }
+    const structuredDisplayPoint = actionDisplayPoint?.status === 'ok'
+      ? actionDisplayPoint.structured
+      : undefined
+
     if (decision.requiresApproval && !options.skipApprovalQueue) {
       const pending = runtime.session.createPendingAction({
         toolName,
@@ -380,6 +495,7 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
           backendResult = {
             ...result,
             pointerTrace,
+            displayPoint: structuredDisplayPoint,
           }
           break
         }
@@ -406,6 +522,7 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
               })
               runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y })
               backendResult.focusPointerTrace = pointerTrace
+              backendResult.focusDisplayPoint = structuredDisplayPoint
             }
             catch (clickError) {
               const msg = clickError instanceof Error ? clickError.message : String(clickError)
@@ -479,7 +596,10 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
           if (typeof normalizedAction.input.x === 'number' && typeof normalizedAction.input.y === 'number') {
             runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y })
           }
-          backendResult = { ...result }
+          backendResult = {
+            ...result,
+            displayPoint: structuredDisplayPoint,
+          }
           break
         }
         case 'wait': {

services/computer-use-mcp/src/server/register-tools-coordinate-contract.test.ts

@@ -0,0 +1,65 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
+
+import type { ComputerUseServerRuntime } from './runtime'
+
+import { describe, expect, it, vi } from 'vitest'
+
+import { RunStateManager } from '../state'
+import { createTestConfig } from '../test-fixtures'
+import { registerComputerUseTools } from './register-tools'
+
+type SchemaShape = Record<string, { description?: string }>
+
+function createMockServer() {
+  const schemas = new Map<string, SchemaShape>()
+
+  return {
+    server: {
+      tool(name: string, schemaOrSummary: unknown, schemaOrHandler?: unknown) {
+        const schema = typeof schemaOrSummary === 'string'
+          ? schemaOrHandler
+          : schemaOrSummary
+
+        schemas.set(name, schema as SchemaShape)
+
+        return { disable: vi.fn() }
+      },
+    } as unknown as McpServer,
+    schemas,
+  }
+}
+
+function createRegistrationRuntime() {
+  return {
+    config: createTestConfig(),
+    stateManager: new RunStateManager(),
+    session: {},
+    executor: {},
+    terminalRunner: {},
+    browserDomBridge: {},
+    cdpBridgeManager: {},
+    chromeSessionManager: {},
+    desktopSessionController: {},
+    taskMemory: {},
+  } as unknown as ComputerUseServerRuntime
+}
+
+describe('registerComputerUseTools coordinate contract', () => {
+  it('documents mutating desktop target coordinates as global logical screen coordinates', () => {
+    const { server, schemas } = createMockServer()
+
+    registerComputerUseTools({
+      server,
+      runtime: createRegistrationRuntime(),
+      executeAction: vi.fn(),
+      enableTestTools: false,
+    })
+
+    expect(schemas.get('desktop_click')?.x.description).toBe('Global logical screen X coordinate, not Retina backing pixels')
+    expect(schemas.get('desktop_click')?.y.description).toBe('Global logical screen Y coordinate, not Retina backing pixels')
+    expect(schemas.get('desktop_type_text')?.x.description).toBe('Optional global logical screen X coordinate to click before typing')
+    expect(schemas.get('desktop_type_text')?.y.description).toBe('Optional global logical screen Y coordinate to click before typing')
+    expect(schemas.get('desktop_scroll')?.x.description).toBe('Optional global logical screen X coordinate to move to before scrolling')
+    expect(schemas.get('desktop_scroll')?.y.description).toBe('Optional global logical screen Y coordinate to move to before scrolling')
+  })
+})

services/computer-use-mcp/src/server/register-tools.ts

@@ -242,8 +242,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
   server.tool(
     'desktop_click',
     {
-      x: z.number().describe('Absolute screen X coordinate in pixels'),
-      y: z.number().describe('Absolute screen Y coordinate in pixels'),
+      x: z.number().describe('Global logical screen X coordinate, not Retina backing pixels'),
+      y: z.number().describe('Global logical screen Y coordinate, not Retina backing pixels'),
       button: z.enum(['left', 'right', 'middle']).optional().describe('Mouse button, default left'),
       clickCount: z.number().int().min(1).max(2).optional().describe('Number of clicks, default 1'),
       captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'),
@@ -255,8 +255,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
     'desktop_type_text',
     {
       text: z.string().min(1).describe('Text to type into the focused UI element'),
-      x: z.number().optional().describe('Optional X coordinate to click before typing'),
-      y: z.number().optional().describe('Optional Y coordinate to click before typing'),
+      x: z.number().optional().describe('Optional global logical screen X coordinate to click before typing'),
+      y: z.number().optional().describe('Optional global logical screen Y coordinate to click before typing'),
       pressEnter: z.boolean().optional().describe('Whether to press Enter after typing'),
       captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'),
     },
@@ -275,8 +275,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
   server.tool(
     'desktop_scroll',
     {
-      x: z.number().optional().describe('Optional X coordinate to move to before scrolling'),
-      y: z.number().optional().describe('Optional Y coordinate to move to before scrolling'),
+      x: z.number().optional().describe('Optional global logical screen X coordinate to move to before scrolling'),
+      y: z.number().optional().describe('Optional global logical screen Y coordinate to move to before scrolling'),
       deltaX: z.number().optional().describe('Horizontal scroll delta in pixels'),
       deltaY: z.number().describe('Vertical scroll delta in pixels'),
       captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'),

services/computer-use-mcp/src/server/tool-descriptors/desktop.ts

@@ -117,7 +117,7 @@ export const desktopDescriptors: ToolDescriptor[] = [
   {
     canonicalName: 'desktop_click',
     displayName: 'Desktop Click',
-    summary: 'Click at screen coordinates. Supports left/right/middle buttons and click count for double-click.',
+    summary: 'Click at global logical screen coordinates, not Retina backing pixels. Supports left/right/middle buttons and click count for double-click.',
     lane: 'desktop',
     kind: 'write',
     readOnly: false,
@@ -130,7 +130,7 @@ export const desktopDescriptors: ToolDescriptor[] = [
   {
     canonicalName: 'desktop_type_text',
     displayName: 'Desktop Type Text',
-    summary: 'Type text into the focused element. Optionally click at coordinates first. Long text (>160 chars) requires approval.',
+    summary: 'Type text into the focused element. Optionally click at global logical screen coordinates first. Long text (>160 chars) requires approval.',
     lane: 'desktop',
     kind: 'write',
     readOnly: false,
@@ -156,7 +156,7 @@ export const desktopDescriptors: ToolDescriptor[] = [
   {
     canonicalName: 'desktop_scroll',
     displayName: 'Desktop Scroll',
-    summary: 'Scroll the screen at optional coordinates. Supports both vertical and horizontal scrolling.',
+    summary: 'Scroll the screen at optional global logical screen coordinates. Supports both vertical and horizontal scrolling.',
     lane: 'desktop',
     kind: 'write',
     readOnly: false,

services/computer-use-mcp/src/server/tool-descriptors/registry.test.ts

@@ -296,6 +296,15 @@ describe('toolDescriptorRegistry', () => {
       expect(desc.readOnly).toBe(true)
       expect(desc.public).toBe(true)
     })
+
+    it('documents direct desktop coordinate tools as global logical coordinates', () => {
+      const registry = createPopulatedRegistry()
+
+      expect(registry.get('desktop_click').summary).toContain('global logical screen coordinates')
+      expect(registry.get('desktop_click').summary).toContain('not Retina backing pixels')
+      expect(registry.get('desktop_type_text').summary).toContain('global logical screen coordinates')
+      expect(registry.get('desktop_scroll').summary).toContain('global logical screen coordinates')
+    })
   })
 
   describe('desktop grounding tool enablement', () => {

services/computer-use-mcp/src/types.ts

@@ -191,7 +191,9 @@ export interface PointerTracePoint {
 }
 
 export interface ClickActionInput {
+  /** Global logical screen coordinate, not Retina backing pixels. */
   x: number
+  /** Global logical screen coordinate, not Retina backing pixels. */
   y: number
   button?: MouseButton
   clickCount?: number
@@ -200,7 +202,9 @@ export interface ClickActionInput {
 
 export interface TypeTextActionInput {
   text: string
+  /** Optional global logical screen coordinate to focus before typing. */
   x?: number
+  /** Optional global logical screen coordinate to focus before typing. */
   y?: number
   pressEnter?: boolean
   captureAfter?: boolean
@@ -212,7 +216,9 @@ export interface PressKeysActionInput {
 }
 
 export interface ScrollActionInput {
+  /** Optional global logical screen coordinate to move to before scrolling. */
   x?: number
+  /** Optional global logical screen coordinate to move to before scrolling. */
   y?: number
   deltaX?: number
   deltaY: number