diff --git a/services/computer-use-mcp/src/display/display.test.ts b/services/computer-use-mcp/src/display/display.test.ts index b2087a9b9..426e4daf7 100644 --- a/services/computer-use-mcp/src/display/display.test.ts +++ b/services/computer-use-mcp/src/display/display.test.ts @@ -2,34 +2,34 @@ import type { DisplayDescriptor, MultiDisplaySnapshot } from '../display/types' import { describe, expect, it } from 'vitest' -import { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from '../display/types' +import { findDisplayForPoint, resolveDisplayPoint, toBackingPixelCoord, toDisplayLocalCoord, toGlobalCoord } from '../display/types' function createTestSnapshot(): MultiDisplaySnapshot { const mainDisplay: DisplayDescriptor = { displayId: 1, isMain: true, isBuiltIn: true, - bounds: { x: 0, y: 0, width: 1440, height: 900 }, - visibleBounds: { x: 0, y: 25, width: 1440, height: 875 }, + bounds: { x: 0, y: 0, width: 1512, height: 982 }, + visibleBounds: { x: 0, y: 65, width: 1512, height: 884 }, scaleFactor: 2, - pixelWidth: 2880, - pixelHeight: 1800, + pixelWidth: 3024, + pixelHeight: 1964, } const externalDisplay: DisplayDescriptor = { - displayId: 2, + displayId: 3, isMain: false, isBuiltIn: false, - bounds: { x: 1440, y: 0, width: 2560, height: 1440 }, - visibleBounds: { x: 1440, y: 25, width: 2560, height: 1415 }, + bounds: { x: -222, y: -1080, width: 1920, height: 1080 }, + visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 }, scaleFactor: 1, - pixelWidth: 2560, - pixelHeight: 1440, + pixelWidth: 1920, + pixelHeight: 1080, } return { displays: [mainDisplay, externalDisplay], - combinedBounds: { x: 0, y: 0, width: 4000, height: 1440 }, + combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 }, capturedAt: '2025-01-01T00:00:00.000Z', } } @@ -37,7 +37,7 @@ function createTestSnapshot(): MultiDisplaySnapshot { describe('findDisplayForPoint', () => { it('finds the main display for a point on it', () => { const snapshot = createTestSnapshot() - const display = findDisplayForPoint(snapshot, 720, 450) + const display = findDisplayForPoint(snapshot, 756, 491) expect(display).toBeDefined() expect(display!.displayId).toBe(1) @@ -46,10 +46,10 @@ describe('findDisplayForPoint', () => { it('finds the external display for a point on it', () => { const snapshot = createTestSnapshot() - const display = findDisplayForPoint(snapshot, 2000, 500) + const display = findDisplayForPoint(snapshot, -100, -500) expect(display).toBeDefined() - expect(display!.displayId).toBe(2) + expect(display!.displayId).toBe(3) expect(display!.isMain).toBe(false) }) @@ -62,53 +62,53 @@ describe('findDisplayForPoint', () => { it('handles edge case at display boundary', () => { const snapshot = createTestSnapshot() - // x=1440 is the first pixel of the external display - const display = findDisplayForPoint(snapshot, 1440, 0) + // y=0 is the first row of the main display below the upper external display. + const display = findDisplayForPoint(snapshot, 0, 0) expect(display).toBeDefined() - expect(display!.displayId).toBe(2) + expect(display!.displayId).toBe(1) }) }) describe('toDisplayLocalCoord', () => { it('converts global to local coordinates on main display', () => { const snapshot = createTestSnapshot() - const local = toDisplayLocalCoord(snapshot.displays[0], 720, 450) + const local = toDisplayLocalCoord(snapshot.displays[0], 756, 491) - expect(local.x).toBe(720) - expect(local.y).toBe(450) + expect(local.x).toBe(756) + expect(local.y).toBe(491) }) it('converts global to local coordinates on external display', () => { const snapshot = createTestSnapshot() - const local = toDisplayLocalCoord(snapshot.displays[1], 2000, 500) + const local = toDisplayLocalCoord(snapshot.displays[1], -100, -500) - expect(local.x).toBe(560) // 2000 - 1440 - expect(local.y).toBe(500) + expect(local.x).toBe(122) + expect(local.y).toBe(580) }) }) describe('toGlobalCoord', () => { it('converts local back to global on main display', () => { const snapshot = createTestSnapshot() - const global = toGlobalCoord(snapshot.displays[0], 720, 450) + const global = toGlobalCoord(snapshot.displays[0], 756, 491) - expect(global.x).toBe(720) - expect(global.y).toBe(450) + expect(global.x).toBe(756) + expect(global.y).toBe(491) }) it('converts local back to global on external display', () => { const snapshot = createTestSnapshot() - const global = toGlobalCoord(snapshot.displays[1], 560, 500) + const global = toGlobalCoord(snapshot.displays[1], 122, 580) - expect(global.x).toBe(2000) - expect(global.y).toBe(500) + expect(global.x).toBe(-100) + expect(global.y).toBe(-500) }) it('is inverse of toDisplayLocalCoord', () => { const snapshot = createTestSnapshot() const display = snapshot.displays[1] - const globalPoint = { x: 2500, y: 800 } + const globalPoint = { x: 800, y: -100 } const local = toDisplayLocalCoord(display, globalPoint.x, globalPoint.y) const roundTrip = toGlobalCoord(display, local.x, local.y) @@ -116,3 +116,43 @@ describe('toGlobalCoord', () => { expect(roundTrip.y).toBe(globalPoint.y) }) }) + +describe('toBackingPixelCoord', () => { + it('maps Retina display-local logical coordinates to backing pixels', () => { + const snapshot = createTestSnapshot() + const backing = toBackingPixelCoord(snapshot.displays[0], 100, 50) + + expect(backing).toEqual({ x: 200, y: 100 }) + }) + + it('leaves 1x external display-local coordinates unchanged', () => { + const snapshot = createTestSnapshot() + const backing = toBackingPixelCoord(snapshot.displays[1], 122, 580) + + expect(backing).toEqual({ x: 122, y: 580 }) + }) +}) + +describe('resolveDisplayPoint', () => { + it('resolves main Retina global point without scaling the original global coordinate', () => { + const snapshot = createTestSnapshot() + const resolved = resolveDisplayPoint(snapshot, 100, 50) + + expect(resolved).toBeDefined() + expect(resolved!.global).toEqual({ x: 100, y: 50 }) + expect(resolved!.local).toEqual({ x: 100, y: 50 }) + expect(resolved!.backingPixel).toEqual({ x: 200, y: 100 }) + expect(resolved!.display.displayId).toBe(1) + }) + + it('resolves negative-coordinate external display points', () => { + const snapshot = createTestSnapshot() + const resolved = resolveDisplayPoint(snapshot, -100, -500) + + expect(resolved).toBeDefined() + expect(resolved!.global).toEqual({ x: -100, y: -500 }) + expect(resolved!.local).toEqual({ x: 122, y: 580 }) + expect(resolved!.backingPixel).toEqual({ x: 122, y: 580 }) + expect(resolved!.display.displayId).toBe(3) + }) +}) diff --git a/services/computer-use-mcp/src/display/index.ts b/services/computer-use-mcp/src/display/index.ts index 3334bab27..f6592faac 100644 --- a/services/computer-use-mcp/src/display/index.ts +++ b/services/computer-use-mcp/src/display/index.ts @@ -1,3 +1,13 @@ export { enumerateDisplays, formatDisplaySummary } from './enumerate' -export { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from './types' -export type { DisplayDescriptor, MultiDisplaySnapshot } from './types' +export { + findDisplayForPoint, + resolveDisplayPoint, + toBackingPixelCoord, + toDisplayLocalCoord, + toGlobalCoord, +} from './types' +export type { + DisplayDescriptor, + DisplayPointResolution, + MultiDisplaySnapshot, +} from './types' diff --git a/services/computer-use-mcp/src/display/types.ts b/services/computer-use-mcp/src/display/types.ts index de3cb3cd0..a399f2759 100644 --- a/services/computer-use-mcp/src/display/types.ts +++ b/services/computer-use-mcp/src/display/types.ts @@ -42,6 +42,33 @@ export interface MultiDisplaySnapshot { capturedAt: string } +/** + * Resolved position metadata for a point in AIRI's desktop coordinate space. + * + * The `global` coordinate is the unscaled logical point that macOS input events + * should receive. `local` and `backingPixel` are diagnostics for display-aware + * rendering, overlays, and Retina mismatch debugging. + */ +export interface DisplayPointResolution { + /** Original point in global logical screen coordinates. */ + global: { + x: number + y: number + } + /** Display containing the global logical point. */ + display: DisplayDescriptor + /** Point relative to the containing display in logical coordinates. */ + local: { + x: number + y: number + } + /** Point relative to the containing display in backing pixels. */ + backingPixel: { + x: number + y: number + } +} + /** * Given a logical screen point, find which display it belongs to. */ @@ -56,6 +83,30 @@ export function findDisplayForPoint( }) } +/** + * Converts display-local logical coordinates to backing pixels. + * + * Use when: + * - Rendering display-local overlay diagnostics + * - Comparing logical desktop points with backing-pixel screenshots + * + * Expects: + * - `localX` and `localY` are already relative to `display.bounds` + * + * Returns: + * - Display-local backing-pixel coordinates rounded to integer pixels + */ +export function toBackingPixelCoord( + display: DisplayDescriptor, + localX: number, + localY: number, +): { x: number, y: number } { + return { + x: Math.round(localX * display.scaleFactor), + y: Math.round(localY * display.scaleFactor), + } +} + /** * Convert a logical coordinate to the local coordinate space of a specific display. */ @@ -70,6 +121,41 @@ export function toDisplayLocalCoord( } } +/** + * Resolves a global logical point against the display snapshot. + * + * Use when: + * - Mapping desktop mutation targets to a concrete macOS display + * - Recording display-local and backing-pixel diagnostics + * + * Expects: + * - `snapshot` uses AIRI's top-left global logical coordinate space + * - `x` and `y` are not pre-scaled by Retina backing factor + * + * Returns: + * - Display metadata when the point is inside a connected display + * - `undefined` when the point is outside all display bounds + */ +export function resolveDisplayPoint( + snapshot: MultiDisplaySnapshot, + x: number, + y: number, +): DisplayPointResolution | undefined { + const display = findDisplayForPoint(snapshot, x, y) + if (!display) { + return undefined + } + + const local = toDisplayLocalCoord(display, x, y) + + return { + global: { x, y }, + display, + local, + backingPixel: toBackingPixelCoord(display, local.x, local.y), + } +} + /** * Convert display-local coordinates back to global logical coordinates. */ diff --git a/services/computer-use-mcp/src/runtime-probes.test.ts b/services/computer-use-mcp/src/runtime-probes.test.ts index 01d6a9378..d38babd8b 100644 --- a/services/computer-use-mcp/src/runtime-probes.test.ts +++ b/services/computer-use-mcp/src/runtime-probes.test.ts @@ -1,14 +1,44 @@ +import type { MultiDisplaySnapshot } from './display' import type { ComputerUseConfig } from './types' import { describe, expect, it } from 'vitest' -import { buildCoordinateSpaceInfo } from './runtime-probes' +import { buildCoordinateSpaceInfo, buildDisplayInfoFromSnapshot } from './runtime-probes' import { createTestConfig } from './test-fixtures' const baseConfig: ComputerUseConfig = createTestConfig({ allowedBounds: { x: 0, y: 0, width: 1440, height: 900 }, }) +function createMultiDisplaySnapshot(): MultiDisplaySnapshot { + return { + displays: [ + { + displayId: 1, + isMain: true, + isBuiltIn: true, + bounds: { x: 0, y: 0, width: 1512, height: 982 }, + visibleBounds: { x: 0, y: 65, width: 1512, height: 884 }, + scaleFactor: 2, + pixelWidth: 3024, + pixelHeight: 1964, + }, + { + displayId: 3, + isMain: false, + isBuiltIn: false, + bounds: { x: -222, y: -1080, width: 1920, height: 1080 }, + visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 }, + scaleFactor: 1, + pixelWidth: 1920, + pixelHeight: 1080, + }, + ], + combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 }, + capturedAt: '2026-04-27T00:00:00.000Z', + } +} + describe('buildCoordinateSpaceInfo', () => { it('requires a screenshot before real input', () => { const info = buildCoordinateSpaceInfo({ @@ -59,4 +89,39 @@ describe('buildCoordinateSpaceInfo', () => { expect(info.aligned).toBe(false) expect(info.reason).toContain('Retina') }) + + it('keeps allowed bounds valid when they sit inside combined multi-display bounds', () => { + const info = buildCoordinateSpaceInfo({ + config: createTestConfig({ + allowedBounds: { x: -222, y: -1080, width: 1920, height: 2062 }, + }), + lastScreenshot: { + path: '/tmp/screenshot.png', + width: 1920, + height: 2062, + placeholder: false, + }, + displayInfo: buildDisplayInfoFromSnapshot(createMultiDisplaySnapshot(), 'darwin'), + }) + + expect(info.readyForMutations).toBe(true) + expect(info.aligned).toBe(true) + }) +}) + +describe('buildDisplayInfoFromSnapshot', () => { + it('preserves legacy main-display facts while exposing multi-display bounds', () => { + const info = buildDisplayInfoFromSnapshot(createMultiDisplaySnapshot(), 'darwin') + + expect(info.available).toBe(true) + expect(info.logicalWidth).toBe(1512) + expect(info.logicalHeight).toBe(982) + expect(info.pixelWidth).toBe(3024) + expect(info.pixelHeight).toBe(1964) + expect(info.scaleFactor).toBe(2) + expect(info.isRetina).toBe(true) + expect(info.displayCount).toBe(2) + expect(info.displays?.[1]?.bounds).toEqual({ x: -222, y: -1080, width: 1920, height: 1080 }) + expect(info.combinedBounds).toEqual({ x: -222, y: -1080, width: 1920, height: 2062 }) + }) }) diff --git a/services/computer-use-mcp/src/runtime-probes.ts b/services/computer-use-mcp/src/runtime-probes.ts index ab42fd855..c4e89ce97 100644 --- a/services/computer-use-mcp/src/runtime-probes.ts +++ b/services/computer-use-mcp/src/runtime-probes.ts @@ -1,3 +1,4 @@ +import type { MultiDisplaySnapshot } from './display' import type { ComputerUseConfig, CoordinateSpaceInfo, @@ -12,6 +13,7 @@ import { hostname } from 'node:os' import { basename } from 'node:path' import { argv, pid, platform, ppid, title } from 'node:process' +import { enumerateDisplays } from './display' import { runProcess } from './utils/process' import { runSwiftScript } from './utils/swift' @@ -43,6 +45,53 @@ export function resolveLaunchContext(config: ComputerUseConfig): LaunchContext { } } +/** + * Builds the public runtime display facts from the native display snapshot. + * + * Use when: + * - Publishing display facts through desktop capabilities/state + * - Preserving legacy main-display fields while exposing multi-display bounds + * + * Expects: + * - `snapshot` uses AIRI's top-left global logical coordinate space + * + * Returns: + * - DisplayInfo with legacy main-display fields and complete display list + */ +export function buildDisplayInfoFromSnapshot( + snapshot: MultiDisplaySnapshot, + targetPlatform: NodeJS.Platform = platform, +): DisplayInfo { + const mainDisplay = snapshot.displays.find(display => display.isMain) ?? snapshot.displays[0] + + if (!mainDisplay) { + return { + available: false, + platform: targetPlatform, + displayCount: 0, + displays: [], + combinedBounds: snapshot.combinedBounds, + capturedAt: snapshot.capturedAt, + note: 'display enumeration returned no connected displays', + } + } + + return { + available: true, + platform: targetPlatform, + logicalWidth: mainDisplay.bounds.width, + logicalHeight: mainDisplay.bounds.height, + pixelWidth: mainDisplay.pixelWidth, + pixelHeight: mainDisplay.pixelHeight, + scaleFactor: mainDisplay.scaleFactor, + isRetina: mainDisplay.scaleFactor > 1, + displayCount: snapshot.displays.length, + displays: snapshot.displays, + combinedBounds: snapshot.combinedBounds, + capturedAt: snapshot.capturedAt, + } +} + export async function probeDisplayInfo(config: ComputerUseConfig): Promise { if (platform !== 'darwin') { return { @@ -52,43 +101,8 @@ export async function probeDisplayInfo(config: ComputerUseConfig): Promise 1.0 -] - -let data = try JSONSerialization.data(withJSONObject: payload, options: []) -print(String(data: data, encoding: .utf8)!) -` - try { - const { stdout } = await runSwiftScript({ - swiftBinary: config.binaries.swift, - timeoutMs: config.timeoutMs, - source: script, - }) - const parsed = JSON.parse(stdout.trim()) as Omit - - return { - platform, - ...parsed, - } + return buildDisplayInfoFromSnapshot(await enumerateDisplays(config), platform) } catch (error) { return { diff --git a/services/computer-use-mcp/src/server/action-executor.test.ts b/services/computer-use-mcp/src/server/action-executor.test.ts index 36270c612..1bb1ab98c 100644 --- a/services/computer-use-mcp/src/server/action-executor.test.ts +++ b/services/computer-use-mcp/src/server/action-executor.test.ts @@ -108,6 +108,49 @@ function createRuntimeForActionTest(configOverrides: Partial } } +function createMultiDisplayInfo() { + return createDisplayInfo({ + platform: 'darwin', + logicalWidth: 1512, + logicalHeight: 982, + pixelWidth: 3024, + pixelHeight: 1964, + scaleFactor: 2, + isRetina: true, + displayCount: 2, + displays: [ + { + displayId: 1, + isMain: true, + isBuiltIn: true, + bounds: { x: 0, y: 0, width: 1512, height: 982 }, + visibleBounds: { x: 0, y: 65, width: 1512, height: 884 }, + scaleFactor: 2, + pixelWidth: 3024, + pixelHeight: 1964, + }, + { + displayId: 3, + isMain: false, + isBuiltIn: false, + bounds: { x: -222, y: -1080, width: 1920, height: 1080 }, + visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 }, + scaleFactor: 1, + pixelWidth: 1920, + pixelHeight: 1080, + }, + ], + combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 }, + capturedAt: '2026-04-27T00:00:00.000Z', + }) +} + +function createCombinedDisplayBoundsConfig() { + return { + allowedBounds: { x: -222, y: -1080, width: 1920, height: 2062 }, + } +} + describe('createExecuteAction', () => { it('executes desktop_click_target through the shared policy and audit pipeline', async () => { const { runtime, executor, session, stateManager } = createRuntimeForActionTest() @@ -651,4 +694,99 @@ describe('createExecuteAction', () => { expect(executor.typeText).toHaveBeenCalledOnce() expect(browserDomBridge.setInputValue).not.toHaveBeenCalled() }) + + it('records main-display metadata while preserving original global logical click coordinates', async () => { + const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig()) + executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo()) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'click', input: { x: 100, y: 50, button: 'left', captureAfter: false } }, 'desktop_click') + + expect(result.isError).not.toBe(true) + expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({ + x: 100, + y: 50, + pointerTrace: expect.arrayContaining([ + expect.objectContaining({ x: 100, y: 50 }), + ]), + })) + + const structured = result.structuredContent as Record + expect(structured.backendResult.displayPoint).toMatchObject({ + coordinateSpace: 'global-logical', + global: { x: 100, y: 50 }, + displayId: 1, + local: { x: 100, y: 50 }, + backingPixel: { x: 200, y: 100 }, + scaleFactor: 2, + }) + }) + + it('accepts negative-coordinate external display clicks and records display-local metadata', async () => { + const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig()) + executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo()) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'click', input: { x: -100, y: -500, captureAfter: false } }, 'desktop_click') + + expect(result.isError).not.toBe(true) + expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({ + x: -100, + y: -500, + pointerTrace: expect.arrayContaining([ + expect.objectContaining({ x: -100, y: -500 }), + ]), + })) + + const structured = result.structuredContent as Record + expect(structured.backendResult.displayPoint).toMatchObject({ + coordinateSpace: 'global-logical', + global: { x: -100, y: -500 }, + displayId: 3, + local: { x: 122, y: 580 }, + backingPixel: { x: 122, y: 580 }, + scaleFactor: 1, + }) + }) + + it('rejects physical-pixel-looking Retina coordinates outside the global logical display contract', async () => { + const { runtime, executor } = createRuntimeForActionTest({ + allowedBounds: { x: -10_000, y: -10_000, width: 20_000, height: 20_000 }, + }) + executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo()) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'click', input: { x: 2000, y: 500, captureAfter: false } }, 'desktop_click') + + expect(result.isError).toBe(true) + expect(executor.click).not.toHaveBeenCalled() + expect((result.content[0] as { text: string }).text).toContain('outside connected display bounds') + }) + + it('uses the same display resolver for type_text preparatory clicks', async () => { + const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig()) + executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo()) + executor.typeText.mockResolvedValue({ + performed: true, + backend: 'dry-run' as const, + notes: [], + }) + + const executeAction = createExecuteAction(runtime) + const result = await executeAction({ kind: 'type_text', input: { text: 'hello', x: -100, y: -500, captureAfter: false } }, 'desktop_type_text') + + expect(result.isError).not.toBe(true) + expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({ + x: -100, + y: -500, + })) + expect(executor.typeText).toHaveBeenCalledTimes(1) + + const structured = result.structuredContent as Record + expect(structured.backendResult.focusDisplayPoint).toMatchObject({ + displayId: 3, + local: { x: 122, y: 580 }, + backingPixel: { x: 122, y: 580 }, + }) + }) }) diff --git a/services/computer-use-mcp/src/server/action-executor.ts b/services/computer-use-mcp/src/server/action-executor.ts index b96f0bdae..3bd22598c 100644 --- a/services/computer-use-mcp/src/server/action-executor.ts +++ b/services/computer-use-mcp/src/server/action-executor.ts @@ -1,9 +1,11 @@ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js' +import type { DisplayPointResolution, MultiDisplaySnapshot } from '../display' import type { ActionInvocation, ComputerUseConfig, DesktopExecutor, + DisplayInfo, ForegroundContext, PolicyDecision, ScreenshotArtifact, @@ -15,6 +17,7 @@ import type { ComputerUseServerRuntime } from './runtime' import { normalizeConfiguredAppAction } from '../app-aliases' import { decideBrowserTypeAction } from '../browser-action-router' import { isBrowserDomActionSupported } from '../browser-dom/capabilities' +import { resolveDisplayPoint } from '../display' import { evaluateActionPolicy } from '../policy' import { getRuntimePreflight } from '../preflight' import { buildCoordinateSpaceInfo } from '../runtime-probes' @@ -119,6 +122,91 @@ function toTerminalStateContent(state: TerminalState) { } } +function displaySnapshotFromDisplayInfo(displayInfo: DisplayInfo): MultiDisplaySnapshot | undefined { + if (!displayInfo.displays?.length) { + return undefined + } + + return { + displays: displayInfo.displays.map(display => ({ + displayId: display.displayId, + isMain: display.isMain, + isBuiltIn: display.isBuiltIn, + bounds: display.bounds, + visibleBounds: display.visibleBounds, + scaleFactor: display.scaleFactor, + pixelWidth: display.pixelWidth, + pixelHeight: display.pixelHeight, + })), + combinedBounds: displayInfo.combinedBounds ?? { + x: 0, + y: 0, + width: 0, + height: 0, + }, + capturedAt: displayInfo.capturedAt ?? new Date(0).toISOString(), + } +} + +function getCoordinateMutationTarget(action: ActionInvocation): { x: number, y: number } | undefined { + switch (action.kind) { + case 'click': + return { x: action.input.x, y: action.input.y } + case 'type_text': + if (typeof action.input.x === 'number' && typeof action.input.y === 'number') { + return { x: action.input.x, y: action.input.y } + } + return undefined + case 'scroll': + if (typeof action.input.x === 'number' && typeof action.input.y === 'number') { + return { x: action.input.x, y: action.input.y } + } + return undefined + default: + return undefined + } +} + +function toStructuredDisplayPoint(resolution: DisplayPointResolution) { + return { + coordinateSpace: 'global-logical', + global: resolution.global, + displayId: resolution.display.displayId, + displayBounds: resolution.display.bounds, + local: resolution.local, + backingPixel: resolution.backingPixel, + scaleFactor: resolution.display.scaleFactor, + } +} + +function resolveActionDisplayPoint(action: ActionInvocation, displayInfo: DisplayInfo) { + const target = getCoordinateMutationTarget(action) + const snapshot = displaySnapshotFromDisplayInfo(displayInfo) + + if (!target || !snapshot) { + return undefined + } + + const resolution = resolveDisplayPoint(snapshot, target.x, target.y) + if (!resolution) { + const combined = displayInfo.combinedBounds + return { + status: 'outside' as const, + target, + reason: combined + ? `target point (${target.x}, ${target.y}) is outside connected display bounds ${combined.width}x${combined.height} @ (${combined.x},${combined.y})` + : `target point (${target.x}, ${target.y}) is outside connected display bounds`, + } + } + + return { + status: 'ok' as const, + target, + resolution, + structured: toStructuredDisplayPoint(resolution), + } +} + function getPolicyEvaluationContext(params: { action: ActionInvocation actualContext: ForegroundContext @@ -255,6 +343,33 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA return buildDeniedResponse(decision, context, executionTarget) } + const actionDisplayPoint = resolveActionDisplayPoint(normalizedAction, displayInfo) + if (actionDisplayPoint?.status === 'outside') { + const deniedDecision = buildDeniedDecision({ + decision, + issues: [actionDisplayPoint.reason], + }) + + await runtime.session.record({ + event: 'denied', + toolName, + action: normalizedAction, + context, + policy: deniedDecision, + result: { + executionTarget, + displayInfo, + coordinateSpace: preflight.coordinateSpace, + targetPoint: actionDisplayPoint.target, + }, + }) + + return buildDeniedResponse(deniedDecision, context, executionTarget) + } + const structuredDisplayPoint = actionDisplayPoint?.status === 'ok' + ? actionDisplayPoint.structured + : undefined + if (decision.requiresApproval && !options.skipApprovalQueue) { const pending = runtime.session.createPendingAction({ toolName, @@ -380,6 +495,7 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA backendResult = { ...result, pointerTrace, + displayPoint: structuredDisplayPoint, } break } @@ -406,6 +522,7 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA }) runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y }) backendResult.focusPointerTrace = pointerTrace + backendResult.focusDisplayPoint = structuredDisplayPoint } catch (clickError) { const msg = clickError instanceof Error ? clickError.message : String(clickError) @@ -479,7 +596,10 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA if (typeof normalizedAction.input.x === 'number' && typeof normalizedAction.input.y === 'number') { runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y }) } - backendResult = { ...result } + backendResult = { + ...result, + displayPoint: structuredDisplayPoint, + } break } case 'wait': { diff --git a/services/computer-use-mcp/src/server/register-tools-coordinate-contract.test.ts b/services/computer-use-mcp/src/server/register-tools-coordinate-contract.test.ts new file mode 100644 index 000000000..714ddab4f --- /dev/null +++ b/services/computer-use-mcp/src/server/register-tools-coordinate-contract.test.ts @@ -0,0 +1,65 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' + +import type { ComputerUseServerRuntime } from './runtime' + +import { describe, expect, it, vi } from 'vitest' + +import { RunStateManager } from '../state' +import { createTestConfig } from '../test-fixtures' +import { registerComputerUseTools } from './register-tools' + +type SchemaShape = Record + +function createMockServer() { + const schemas = new Map() + + return { + server: { + tool(name: string, schemaOrSummary: unknown, schemaOrHandler?: unknown) { + const schema = typeof schemaOrSummary === 'string' + ? schemaOrHandler + : schemaOrSummary + + schemas.set(name, schema as SchemaShape) + + return { disable: vi.fn() } + }, + } as unknown as McpServer, + schemas, + } +} + +function createRegistrationRuntime() { + return { + config: createTestConfig(), + stateManager: new RunStateManager(), + session: {}, + executor: {}, + terminalRunner: {}, + browserDomBridge: {}, + cdpBridgeManager: {}, + chromeSessionManager: {}, + desktopSessionController: {}, + taskMemory: {}, + } as unknown as ComputerUseServerRuntime +} + +describe('registerComputerUseTools coordinate contract', () => { + it('documents mutating desktop target coordinates as global logical screen coordinates', () => { + const { server, schemas } = createMockServer() + + registerComputerUseTools({ + server, + runtime: createRegistrationRuntime(), + executeAction: vi.fn(), + enableTestTools: false, + }) + + expect(schemas.get('desktop_click')?.x.description).toBe('Global logical screen X coordinate, not Retina backing pixels') + expect(schemas.get('desktop_click')?.y.description).toBe('Global logical screen Y coordinate, not Retina backing pixels') + expect(schemas.get('desktop_type_text')?.x.description).toBe('Optional global logical screen X coordinate to click before typing') + expect(schemas.get('desktop_type_text')?.y.description).toBe('Optional global logical screen Y coordinate to click before typing') + expect(schemas.get('desktop_scroll')?.x.description).toBe('Optional global logical screen X coordinate to move to before scrolling') + expect(schemas.get('desktop_scroll')?.y.description).toBe('Optional global logical screen Y coordinate to move to before scrolling') + }) +}) diff --git a/services/computer-use-mcp/src/server/register-tools.ts b/services/computer-use-mcp/src/server/register-tools.ts index 7ee862b1d..5aa87884c 100644 --- a/services/computer-use-mcp/src/server/register-tools.ts +++ b/services/computer-use-mcp/src/server/register-tools.ts @@ -242,8 +242,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions server.tool( 'desktop_click', { - x: z.number().describe('Absolute screen X coordinate in pixels'), - y: z.number().describe('Absolute screen Y coordinate in pixels'), + x: z.number().describe('Global logical screen X coordinate, not Retina backing pixels'), + y: z.number().describe('Global logical screen Y coordinate, not Retina backing pixels'), button: z.enum(['left', 'right', 'middle']).optional().describe('Mouse button, default left'), clickCount: z.number().int().min(1).max(2).optional().describe('Number of clicks, default 1'), captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'), @@ -255,8 +255,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions 'desktop_type_text', { text: z.string().min(1).describe('Text to type into the focused UI element'), - x: z.number().optional().describe('Optional X coordinate to click before typing'), - y: z.number().optional().describe('Optional Y coordinate to click before typing'), + x: z.number().optional().describe('Optional global logical screen X coordinate to click before typing'), + y: z.number().optional().describe('Optional global logical screen Y coordinate to click before typing'), pressEnter: z.boolean().optional().describe('Whether to press Enter after typing'), captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'), }, @@ -275,8 +275,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions server.tool( 'desktop_scroll', { - x: z.number().optional().describe('Optional X coordinate to move to before scrolling'), - y: z.number().optional().describe('Optional Y coordinate to move to before scrolling'), + x: z.number().optional().describe('Optional global logical screen X coordinate to move to before scrolling'), + y: z.number().optional().describe('Optional global logical screen Y coordinate to move to before scrolling'), deltaX: z.number().optional().describe('Horizontal scroll delta in pixels'), deltaY: z.number().describe('Vertical scroll delta in pixels'), captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'), diff --git a/services/computer-use-mcp/src/server/tool-descriptors/desktop.ts b/services/computer-use-mcp/src/server/tool-descriptors/desktop.ts index cd3bc66ff..fc4cb85af 100644 --- a/services/computer-use-mcp/src/server/tool-descriptors/desktop.ts +++ b/services/computer-use-mcp/src/server/tool-descriptors/desktop.ts @@ -117,7 +117,7 @@ export const desktopDescriptors: ToolDescriptor[] = [ { canonicalName: 'desktop_click', displayName: 'Desktop Click', - summary: 'Click at screen coordinates. Supports left/right/middle buttons and click count for double-click.', + summary: 'Click at global logical screen coordinates, not Retina backing pixels. Supports left/right/middle buttons and click count for double-click.', lane: 'desktop', kind: 'write', readOnly: false, @@ -130,7 +130,7 @@ export const desktopDescriptors: ToolDescriptor[] = [ { canonicalName: 'desktop_type_text', displayName: 'Desktop Type Text', - summary: 'Type text into the focused element. Optionally click at coordinates first. Long text (>160 chars) requires approval.', + summary: 'Type text into the focused element. Optionally click at global logical screen coordinates first. Long text (>160 chars) requires approval.', lane: 'desktop', kind: 'write', readOnly: false, @@ -156,7 +156,7 @@ export const desktopDescriptors: ToolDescriptor[] = [ { canonicalName: 'desktop_scroll', displayName: 'Desktop Scroll', - summary: 'Scroll the screen at optional coordinates. Supports both vertical and horizontal scrolling.', + summary: 'Scroll the screen at optional global logical screen coordinates. Supports both vertical and horizontal scrolling.', lane: 'desktop', kind: 'write', readOnly: false, diff --git a/services/computer-use-mcp/src/server/tool-descriptors/registry.test.ts b/services/computer-use-mcp/src/server/tool-descriptors/registry.test.ts index a9e2f1949..360a24354 100644 --- a/services/computer-use-mcp/src/server/tool-descriptors/registry.test.ts +++ b/services/computer-use-mcp/src/server/tool-descriptors/registry.test.ts @@ -296,6 +296,15 @@ describe('toolDescriptorRegistry', () => { expect(desc.readOnly).toBe(true) expect(desc.public).toBe(true) }) + + it('documents direct desktop coordinate tools as global logical coordinates', () => { + const registry = createPopulatedRegistry() + + expect(registry.get('desktop_click').summary).toContain('global logical screen coordinates') + expect(registry.get('desktop_click').summary).toContain('not Retina backing pixels') + expect(registry.get('desktop_type_text').summary).toContain('global logical screen coordinates') + expect(registry.get('desktop_scroll').summary).toContain('global logical screen coordinates') + }) }) describe('desktop grounding tool enablement', () => { diff --git a/services/computer-use-mcp/src/types.ts b/services/computer-use-mcp/src/types.ts index 95beba985..0d866f11c 100644 --- a/services/computer-use-mcp/src/types.ts +++ b/services/computer-use-mcp/src/types.ts @@ -191,7 +191,9 @@ export interface PointerTracePoint { } export interface ClickActionInput { + /** Global logical screen coordinate, not Retina backing pixels. */ x: number + /** Global logical screen coordinate, not Retina backing pixels. */ y: number button?: MouseButton clickCount?: number @@ -200,7 +202,9 @@ export interface ClickActionInput { export interface TypeTextActionInput { text: string + /** Optional global logical screen coordinate to focus before typing. */ x?: number + /** Optional global logical screen coordinate to focus before typing. */ y?: number pressEnter?: boolean captureAfter?: boolean @@ -212,7 +216,9 @@ export interface PressKeysActionInput { } export interface ScrollActionInput { + /** Optional global logical screen coordinate to move to before scrolling. */ x?: number + /** Optional global logical screen coordinate to move to before scrolling. */ y?: number deltaX?: number deltaY: number