fix(computer-use-mcp): add macOS display geometry contract (#1750)
--------- Co-authored-by-agent: Antigravity <antigravity@gemini.com>
This commit is contained in:
刘梓恒
@@ -2,34 +2,34 @@ import type { DisplayDescriptor, MultiDisplaySnapshot } from '../display/types'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from '../display/types'
|
||||
import { findDisplayForPoint, resolveDisplayPoint, toBackingPixelCoord, toDisplayLocalCoord, toGlobalCoord } from '../display/types'
|
||||
|
||||
function createTestSnapshot(): MultiDisplaySnapshot {
|
||||
const mainDisplay: DisplayDescriptor = {
|
||||
displayId: 1,
|
||||
isMain: true,
|
||||
isBuiltIn: true,
|
||||
bounds: { x: 0, y: 0, width: 1440, height: 900 },
|
||||
visibleBounds: { x: 0, y: 25, width: 1440, height: 875 },
|
||||
bounds: { x: 0, y: 0, width: 1512, height: 982 },
|
||||
visibleBounds: { x: 0, y: 65, width: 1512, height: 884 },
|
||||
scaleFactor: 2,
|
||||
pixelWidth: 2880,
|
||||
pixelHeight: 1800,
|
||||
pixelWidth: 3024,
|
||||
pixelHeight: 1964,
|
||||
}
|
||||
|
||||
const externalDisplay: DisplayDescriptor = {
|
||||
displayId: 2,
|
||||
displayId: 3,
|
||||
isMain: false,
|
||||
isBuiltIn: false,
|
||||
bounds: { x: 1440, y: 0, width: 2560, height: 1440 },
|
||||
visibleBounds: { x: 1440, y: 25, width: 2560, height: 1415 },
|
||||
bounds: { x: -222, y: -1080, width: 1920, height: 1080 },
|
||||
visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 },
|
||||
scaleFactor: 1,
|
||||
pixelWidth: 2560,
|
||||
pixelHeight: 1440,
|
||||
pixelWidth: 1920,
|
||||
pixelHeight: 1080,
|
||||
}
|
||||
|
||||
return {
|
||||
displays: [mainDisplay, externalDisplay],
|
||||
combinedBounds: { x: 0, y: 0, width: 4000, height: 1440 },
|
||||
combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
|
||||
capturedAt: '2025-01-01T00:00:00.000Z',
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ function createTestSnapshot(): MultiDisplaySnapshot {
|
||||
describe('findDisplayForPoint', () => {
|
||||
it('finds the main display for a point on it', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const display = findDisplayForPoint(snapshot, 720, 450)
|
||||
const display = findDisplayForPoint(snapshot, 756, 491)
|
||||
|
||||
expect(display).toBeDefined()
|
||||
expect(display!.displayId).toBe(1)
|
||||
@@ -46,10 +46,10 @@ describe('findDisplayForPoint', () => {
|
||||
|
||||
it('finds the external display for a point on it', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const display = findDisplayForPoint(snapshot, 2000, 500)
|
||||
const display = findDisplayForPoint(snapshot, -100, -500)
|
||||
|
||||
expect(display).toBeDefined()
|
||||
expect(display!.displayId).toBe(2)
|
||||
expect(display!.displayId).toBe(3)
|
||||
expect(display!.isMain).toBe(false)
|
||||
})
|
||||
|
||||
@@ -62,53 +62,53 @@ describe('findDisplayForPoint', () => {
|
||||
|
||||
it('handles edge case at display boundary', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
// x=1440 is the first pixel of the external display
|
||||
const display = findDisplayForPoint(snapshot, 1440, 0)
|
||||
// y=0 is the first row of the main display below the upper external display.
|
||||
const display = findDisplayForPoint(snapshot, 0, 0)
|
||||
|
||||
expect(display).toBeDefined()
|
||||
expect(display!.displayId).toBe(2)
|
||||
expect(display!.displayId).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('toDisplayLocalCoord', () => {
|
||||
it('converts global to local coordinates on main display', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const local = toDisplayLocalCoord(snapshot.displays[0], 720, 450)
|
||||
const local = toDisplayLocalCoord(snapshot.displays[0], 756, 491)
|
||||
|
||||
expect(local.x).toBe(720)
|
||||
expect(local.y).toBe(450)
|
||||
expect(local.x).toBe(756)
|
||||
expect(local.y).toBe(491)
|
||||
})
|
||||
|
||||
it('converts global to local coordinates on external display', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const local = toDisplayLocalCoord(snapshot.displays[1], 2000, 500)
|
||||
const local = toDisplayLocalCoord(snapshot.displays[1], -100, -500)
|
||||
|
||||
expect(local.x).toBe(560) // 2000 - 1440
|
||||
expect(local.y).toBe(500)
|
||||
expect(local.x).toBe(122)
|
||||
expect(local.y).toBe(580)
|
||||
})
|
||||
})
|
||||
|
||||
describe('toGlobalCoord', () => {
|
||||
it('converts local back to global on main display', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const global = toGlobalCoord(snapshot.displays[0], 720, 450)
|
||||
const global = toGlobalCoord(snapshot.displays[0], 756, 491)
|
||||
|
||||
expect(global.x).toBe(720)
|
||||
expect(global.y).toBe(450)
|
||||
expect(global.x).toBe(756)
|
||||
expect(global.y).toBe(491)
|
||||
})
|
||||
|
||||
it('converts local back to global on external display', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const global = toGlobalCoord(snapshot.displays[1], 560, 500)
|
||||
const global = toGlobalCoord(snapshot.displays[1], 122, 580)
|
||||
|
||||
expect(global.x).toBe(2000)
|
||||
expect(global.y).toBe(500)
|
||||
expect(global.x).toBe(-100)
|
||||
expect(global.y).toBe(-500)
|
||||
})
|
||||
|
||||
it('is inverse of toDisplayLocalCoord', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const display = snapshot.displays[1]
|
||||
const globalPoint = { x: 2500, y: 800 }
|
||||
const globalPoint = { x: 800, y: -100 }
|
||||
const local = toDisplayLocalCoord(display, globalPoint.x, globalPoint.y)
|
||||
const roundTrip = toGlobalCoord(display, local.x, local.y)
|
||||
|
||||
@@ -116,3 +116,43 @@ describe('toGlobalCoord', () => {
|
||||
expect(roundTrip.y).toBe(globalPoint.y)
|
||||
})
|
||||
})
|
||||
|
||||
describe('toBackingPixelCoord', () => {
|
||||
it('maps Retina display-local logical coordinates to backing pixels', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const backing = toBackingPixelCoord(snapshot.displays[0], 100, 50)
|
||||
|
||||
expect(backing).toEqual({ x: 200, y: 100 })
|
||||
})
|
||||
|
||||
it('leaves 1x external display-local coordinates unchanged', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const backing = toBackingPixelCoord(snapshot.displays[1], 122, 580)
|
||||
|
||||
expect(backing).toEqual({ x: 122, y: 580 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveDisplayPoint', () => {
|
||||
it('resolves main Retina global point without scaling the original global coordinate', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const resolved = resolveDisplayPoint(snapshot, 100, 50)
|
||||
|
||||
expect(resolved).toBeDefined()
|
||||
expect(resolved!.global).toEqual({ x: 100, y: 50 })
|
||||
expect(resolved!.local).toEqual({ x: 100, y: 50 })
|
||||
expect(resolved!.backingPixel).toEqual({ x: 200, y: 100 })
|
||||
expect(resolved!.display.displayId).toBe(1)
|
||||
})
|
||||
|
||||
it('resolves negative-coordinate external display points', () => {
|
||||
const snapshot = createTestSnapshot()
|
||||
const resolved = resolveDisplayPoint(snapshot, -100, -500)
|
||||
|
||||
expect(resolved).toBeDefined()
|
||||
expect(resolved!.global).toEqual({ x: -100, y: -500 })
|
||||
expect(resolved!.local).toEqual({ x: 122, y: 580 })
|
||||
expect(resolved!.backingPixel).toEqual({ x: 122, y: 580 })
|
||||
expect(resolved!.display.displayId).toBe(3)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
export { enumerateDisplays, formatDisplaySummary } from './enumerate'
|
||||
export { findDisplayForPoint, toDisplayLocalCoord, toGlobalCoord } from './types'
|
||||
export type { DisplayDescriptor, MultiDisplaySnapshot } from './types'
|
||||
export {
|
||||
findDisplayForPoint,
|
||||
resolveDisplayPoint,
|
||||
toBackingPixelCoord,
|
||||
toDisplayLocalCoord,
|
||||
toGlobalCoord,
|
||||
} from './types'
|
||||
export type {
|
||||
DisplayDescriptor,
|
||||
DisplayPointResolution,
|
||||
MultiDisplaySnapshot,
|
||||
} from './types'
|
||||
|
||||
@@ -42,6 +42,33 @@ export interface MultiDisplaySnapshot {
|
||||
capturedAt: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolved position metadata for a point in AIRI's desktop coordinate space.
|
||||
*
|
||||
* The `global` coordinate is the unscaled logical point that macOS input events
|
||||
* should receive. `local` and `backingPixel` are diagnostics for display-aware
|
||||
* rendering, overlays, and Retina mismatch debugging.
|
||||
*/
|
||||
export interface DisplayPointResolution {
|
||||
/** Original point in global logical screen coordinates. */
|
||||
global: {
|
||||
x: number
|
||||
y: number
|
||||
}
|
||||
/** Display containing the global logical point. */
|
||||
display: DisplayDescriptor
|
||||
/** Point relative to the containing display in logical coordinates. */
|
||||
local: {
|
||||
x: number
|
||||
y: number
|
||||
}
|
||||
/** Point relative to the containing display in backing pixels. */
|
||||
backingPixel: {
|
||||
x: number
|
||||
y: number
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a logical screen point, find which display it belongs to.
|
||||
*/
|
||||
@@ -56,6 +83,30 @@ export function findDisplayForPoint(
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts display-local logical coordinates to backing pixels.
|
||||
*
|
||||
* Use when:
|
||||
* - Rendering display-local overlay diagnostics
|
||||
* - Comparing logical desktop points with backing-pixel screenshots
|
||||
*
|
||||
* Expects:
|
||||
* - `localX` and `localY` are already relative to `display.bounds`
|
||||
*
|
||||
* Returns:
|
||||
* - Display-local backing-pixel coordinates rounded to integer pixels
|
||||
*/
|
||||
export function toBackingPixelCoord(
|
||||
display: DisplayDescriptor,
|
||||
localX: number,
|
||||
localY: number,
|
||||
): { x: number, y: number } {
|
||||
return {
|
||||
x: Math.round(localX * display.scaleFactor),
|
||||
y: Math.round(localY * display.scaleFactor),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a logical coordinate to the local coordinate space of a specific display.
|
||||
*/
|
||||
@@ -70,6 +121,41 @@ export function toDisplayLocalCoord(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a global logical point against the display snapshot.
|
||||
*
|
||||
* Use when:
|
||||
* - Mapping desktop mutation targets to a concrete macOS display
|
||||
* - Recording display-local and backing-pixel diagnostics
|
||||
*
|
||||
* Expects:
|
||||
* - `snapshot` uses AIRI's top-left global logical coordinate space
|
||||
* - `x` and `y` are not pre-scaled by Retina backing factor
|
||||
*
|
||||
* Returns:
|
||||
* - Display metadata when the point is inside a connected display
|
||||
* - `undefined` when the point is outside all display bounds
|
||||
*/
|
||||
export function resolveDisplayPoint(
|
||||
snapshot: MultiDisplaySnapshot,
|
||||
x: number,
|
||||
y: number,
|
||||
): DisplayPointResolution | undefined {
|
||||
const display = findDisplayForPoint(snapshot, x, y)
|
||||
if (!display) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const local = toDisplayLocalCoord(display, x, y)
|
||||
|
||||
return {
|
||||
global: { x, y },
|
||||
display,
|
||||
local,
|
||||
backingPixel: toBackingPixelCoord(display, local.x, local.y),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert display-local coordinates back to global logical coordinates.
|
||||
*/
|
||||
|
||||
@@ -1,14 +1,44 @@
|
||||
import type { MultiDisplaySnapshot } from './display'
|
||||
import type { ComputerUseConfig } from './types'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { buildCoordinateSpaceInfo } from './runtime-probes'
|
||||
import { buildCoordinateSpaceInfo, buildDisplayInfoFromSnapshot } from './runtime-probes'
|
||||
import { createTestConfig } from './test-fixtures'
|
||||
|
||||
const baseConfig: ComputerUseConfig = createTestConfig({
|
||||
allowedBounds: { x: 0, y: 0, width: 1440, height: 900 },
|
||||
})
|
||||
|
||||
function createMultiDisplaySnapshot(): MultiDisplaySnapshot {
|
||||
return {
|
||||
displays: [
|
||||
{
|
||||
displayId: 1,
|
||||
isMain: true,
|
||||
isBuiltIn: true,
|
||||
bounds: { x: 0, y: 0, width: 1512, height: 982 },
|
||||
visibleBounds: { x: 0, y: 65, width: 1512, height: 884 },
|
||||
scaleFactor: 2,
|
||||
pixelWidth: 3024,
|
||||
pixelHeight: 1964,
|
||||
},
|
||||
{
|
||||
displayId: 3,
|
||||
isMain: false,
|
||||
isBuiltIn: false,
|
||||
bounds: { x: -222, y: -1080, width: 1920, height: 1080 },
|
||||
visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 },
|
||||
scaleFactor: 1,
|
||||
pixelWidth: 1920,
|
||||
pixelHeight: 1080,
|
||||
},
|
||||
],
|
||||
combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
|
||||
capturedAt: '2026-04-27T00:00:00.000Z',
|
||||
}
|
||||
}
|
||||
|
||||
describe('buildCoordinateSpaceInfo', () => {
|
||||
it('requires a screenshot before real input', () => {
|
||||
const info = buildCoordinateSpaceInfo({
|
||||
@@ -59,4 +89,39 @@ describe('buildCoordinateSpaceInfo', () => {
|
||||
expect(info.aligned).toBe(false)
|
||||
expect(info.reason).toContain('Retina')
|
||||
})
|
||||
|
||||
it('keeps allowed bounds valid when they sit inside combined multi-display bounds', () => {
|
||||
const info = buildCoordinateSpaceInfo({
|
||||
config: createTestConfig({
|
||||
allowedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
|
||||
}),
|
||||
lastScreenshot: {
|
||||
path: '/tmp/screenshot.png',
|
||||
width: 1920,
|
||||
height: 2062,
|
||||
placeholder: false,
|
||||
},
|
||||
displayInfo: buildDisplayInfoFromSnapshot(createMultiDisplaySnapshot(), 'darwin'),
|
||||
})
|
||||
|
||||
expect(info.readyForMutations).toBe(true)
|
||||
expect(info.aligned).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildDisplayInfoFromSnapshot', () => {
|
||||
it('preserves legacy main-display facts while exposing multi-display bounds', () => {
|
||||
const info = buildDisplayInfoFromSnapshot(createMultiDisplaySnapshot(), 'darwin')
|
||||
|
||||
expect(info.available).toBe(true)
|
||||
expect(info.logicalWidth).toBe(1512)
|
||||
expect(info.logicalHeight).toBe(982)
|
||||
expect(info.pixelWidth).toBe(3024)
|
||||
expect(info.pixelHeight).toBe(1964)
|
||||
expect(info.scaleFactor).toBe(2)
|
||||
expect(info.isRetina).toBe(true)
|
||||
expect(info.displayCount).toBe(2)
|
||||
expect(info.displays?.[1]?.bounds).toEqual({ x: -222, y: -1080, width: 1920, height: 1080 })
|
||||
expect(info.combinedBounds).toEqual({ x: -222, y: -1080, width: 1920, height: 2062 })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { MultiDisplaySnapshot } from './display'
|
||||
import type {
|
||||
ComputerUseConfig,
|
||||
CoordinateSpaceInfo,
|
||||
@@ -12,6 +13,7 @@ import { hostname } from 'node:os'
|
||||
import { basename } from 'node:path'
|
||||
import { argv, pid, platform, ppid, title } from 'node:process'
|
||||
|
||||
import { enumerateDisplays } from './display'
|
||||
import { runProcess } from './utils/process'
|
||||
import { runSwiftScript } from './utils/swift'
|
||||
|
||||
@@ -43,6 +45,53 @@ export function resolveLaunchContext(config: ComputerUseConfig): LaunchContext {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the public runtime display facts from the native display snapshot.
|
||||
*
|
||||
* Use when:
|
||||
* - Publishing display facts through desktop capabilities/state
|
||||
* - Preserving legacy main-display fields while exposing multi-display bounds
|
||||
*
|
||||
* Expects:
|
||||
* - `snapshot` uses AIRI's top-left global logical coordinate space
|
||||
*
|
||||
* Returns:
|
||||
* - DisplayInfo with legacy main-display fields and complete display list
|
||||
*/
|
||||
export function buildDisplayInfoFromSnapshot(
|
||||
snapshot: MultiDisplaySnapshot,
|
||||
targetPlatform: NodeJS.Platform = platform,
|
||||
): DisplayInfo {
|
||||
const mainDisplay = snapshot.displays.find(display => display.isMain) ?? snapshot.displays[0]
|
||||
|
||||
if (!mainDisplay) {
|
||||
return {
|
||||
available: false,
|
||||
platform: targetPlatform,
|
||||
displayCount: 0,
|
||||
displays: [],
|
||||
combinedBounds: snapshot.combinedBounds,
|
||||
capturedAt: snapshot.capturedAt,
|
||||
note: 'display enumeration returned no connected displays',
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
available: true,
|
||||
platform: targetPlatform,
|
||||
logicalWidth: mainDisplay.bounds.width,
|
||||
logicalHeight: mainDisplay.bounds.height,
|
||||
pixelWidth: mainDisplay.pixelWidth,
|
||||
pixelHeight: mainDisplay.pixelHeight,
|
||||
scaleFactor: mainDisplay.scaleFactor,
|
||||
isRetina: mainDisplay.scaleFactor > 1,
|
||||
displayCount: snapshot.displays.length,
|
||||
displays: snapshot.displays,
|
||||
combinedBounds: snapshot.combinedBounds,
|
||||
capturedAt: snapshot.capturedAt,
|
||||
}
|
||||
}
|
||||
|
||||
export async function probeDisplayInfo(config: ComputerUseConfig): Promise<DisplayInfo> {
|
||||
if (platform !== 'darwin') {
|
||||
return {
|
||||
@@ -52,43 +101,8 @@ export async function probeDisplayInfo(config: ComputerUseConfig): Promise<Displ
|
||||
}
|
||||
}
|
||||
|
||||
const script = `
|
||||
import AppKit
|
||||
import Foundation
|
||||
|
||||
guard let screen = NSScreen.main else {
|
||||
print("{\\"available\\":false,\\"note\\":\\"NSScreen.main unavailable\\"}")
|
||||
exit(0)
|
||||
}
|
||||
|
||||
let frame = screen.frame
|
||||
let scale = screen.backingScaleFactor
|
||||
let payload: [String: Any] = [
|
||||
"available": true,
|
||||
"logicalWidth": Int(frame.width),
|
||||
"logicalHeight": Int(frame.height),
|
||||
"pixelWidth": Int(frame.width * scale),
|
||||
"pixelHeight": Int(frame.height * scale),
|
||||
"scaleFactor": scale,
|
||||
"isRetina": scale > 1.0
|
||||
]
|
||||
|
||||
let data = try JSONSerialization.data(withJSONObject: payload, options: [])
|
||||
print(String(data: data, encoding: .utf8)!)
|
||||
`
|
||||
|
||||
try {
|
||||
const { stdout } = await runSwiftScript({
|
||||
swiftBinary: config.binaries.swift,
|
||||
timeoutMs: config.timeoutMs,
|
||||
source: script,
|
||||
})
|
||||
const parsed = JSON.parse(stdout.trim()) as Omit<DisplayInfo, 'platform'>
|
||||
|
||||
return {
|
||||
platform,
|
||||
...parsed,
|
||||
}
|
||||
return buildDisplayInfoFromSnapshot(await enumerateDisplays(config), platform)
|
||||
}
|
||||
catch (error) {
|
||||
return {
|
||||
|
||||
@@ -108,6 +108,49 @@ function createRuntimeForActionTest(configOverrides: Partial<ComputerUseConfig>
|
||||
}
|
||||
}
|
||||
|
||||
function createMultiDisplayInfo() {
|
||||
return createDisplayInfo({
|
||||
platform: 'darwin',
|
||||
logicalWidth: 1512,
|
||||
logicalHeight: 982,
|
||||
pixelWidth: 3024,
|
||||
pixelHeight: 1964,
|
||||
scaleFactor: 2,
|
||||
isRetina: true,
|
||||
displayCount: 2,
|
||||
displays: [
|
||||
{
|
||||
displayId: 1,
|
||||
isMain: true,
|
||||
isBuiltIn: true,
|
||||
bounds: { x: 0, y: 0, width: 1512, height: 982 },
|
||||
visibleBounds: { x: 0, y: 65, width: 1512, height: 884 },
|
||||
scaleFactor: 2,
|
||||
pixelWidth: 3024,
|
||||
pixelHeight: 1964,
|
||||
},
|
||||
{
|
||||
displayId: 3,
|
||||
isMain: false,
|
||||
isBuiltIn: false,
|
||||
bounds: { x: -222, y: -1080, width: 1920, height: 1080 },
|
||||
visibleBounds: { x: -222, y: -1080, width: 1920, height: 1080 },
|
||||
scaleFactor: 1,
|
||||
pixelWidth: 1920,
|
||||
pixelHeight: 1080,
|
||||
},
|
||||
],
|
||||
combinedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
|
||||
capturedAt: '2026-04-27T00:00:00.000Z',
|
||||
})
|
||||
}
|
||||
|
||||
function createCombinedDisplayBoundsConfig() {
|
||||
return {
|
||||
allowedBounds: { x: -222, y: -1080, width: 1920, height: 2062 },
|
||||
}
|
||||
}
|
||||
|
||||
describe('createExecuteAction', () => {
|
||||
it('executes desktop_click_target through the shared policy and audit pipeline', async () => {
|
||||
const { runtime, executor, session, stateManager } = createRuntimeForActionTest()
|
||||
@@ -651,4 +694,99 @@ describe('createExecuteAction', () => {
|
||||
expect(executor.typeText).toHaveBeenCalledOnce()
|
||||
expect(browserDomBridge.setInputValue).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('records main-display metadata while preserving original global logical click coordinates', async () => {
|
||||
const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig())
|
||||
executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
|
||||
|
||||
const executeAction = createExecuteAction(runtime)
|
||||
const result = await executeAction({ kind: 'click', input: { x: 100, y: 50, button: 'left', captureAfter: false } }, 'desktop_click')
|
||||
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({
|
||||
x: 100,
|
||||
y: 50,
|
||||
pointerTrace: expect.arrayContaining([
|
||||
expect.objectContaining({ x: 100, y: 50 }),
|
||||
]),
|
||||
}))
|
||||
|
||||
const structured = result.structuredContent as Record<string, any>
|
||||
expect(structured.backendResult.displayPoint).toMatchObject({
|
||||
coordinateSpace: 'global-logical',
|
||||
global: { x: 100, y: 50 },
|
||||
displayId: 1,
|
||||
local: { x: 100, y: 50 },
|
||||
backingPixel: { x: 200, y: 100 },
|
||||
scaleFactor: 2,
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts negative-coordinate external display clicks and records display-local metadata', async () => {
|
||||
const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig())
|
||||
executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
|
||||
|
||||
const executeAction = createExecuteAction(runtime)
|
||||
const result = await executeAction({ kind: 'click', input: { x: -100, y: -500, captureAfter: false } }, 'desktop_click')
|
||||
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({
|
||||
x: -100,
|
||||
y: -500,
|
||||
pointerTrace: expect.arrayContaining([
|
||||
expect.objectContaining({ x: -100, y: -500 }),
|
||||
]),
|
||||
}))
|
||||
|
||||
const structured = result.structuredContent as Record<string, any>
|
||||
expect(structured.backendResult.displayPoint).toMatchObject({
|
||||
coordinateSpace: 'global-logical',
|
||||
global: { x: -100, y: -500 },
|
||||
displayId: 3,
|
||||
local: { x: 122, y: 580 },
|
||||
backingPixel: { x: 122, y: 580 },
|
||||
scaleFactor: 1,
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects physical-pixel-looking Retina coordinates outside the global logical display contract', async () => {
|
||||
const { runtime, executor } = createRuntimeForActionTest({
|
||||
allowedBounds: { x: -10_000, y: -10_000, width: 20_000, height: 20_000 },
|
||||
})
|
||||
executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
|
||||
|
||||
const executeAction = createExecuteAction(runtime)
|
||||
const result = await executeAction({ kind: 'click', input: { x: 2000, y: 500, captureAfter: false } }, 'desktop_click')
|
||||
|
||||
expect(result.isError).toBe(true)
|
||||
expect(executor.click).not.toHaveBeenCalled()
|
||||
expect((result.content[0] as { text: string }).text).toContain('outside connected display bounds')
|
||||
})
|
||||
|
||||
it('uses the same display resolver for type_text preparatory clicks', async () => {
|
||||
const { runtime, executor } = createRuntimeForActionTest(createCombinedDisplayBoundsConfig())
|
||||
executor.getDisplayInfo.mockResolvedValue(createMultiDisplayInfo())
|
||||
executor.typeText.mockResolvedValue({
|
||||
performed: true,
|
||||
backend: 'dry-run' as const,
|
||||
notes: [],
|
||||
})
|
||||
|
||||
const executeAction = createExecuteAction(runtime)
|
||||
const result = await executeAction({ kind: 'type_text', input: { text: 'hello', x: -100, y: -500, captureAfter: false } }, 'desktop_type_text')
|
||||
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(executor.click).toHaveBeenCalledWith(expect.objectContaining({
|
||||
x: -100,
|
||||
y: -500,
|
||||
}))
|
||||
expect(executor.typeText).toHaveBeenCalledTimes(1)
|
||||
|
||||
const structured = result.structuredContent as Record<string, any>
|
||||
expect(structured.backendResult.focusDisplayPoint).toMatchObject({
|
||||
displayId: 3,
|
||||
local: { x: 122, y: 580 },
|
||||
backingPixel: { x: 122, y: 580 },
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'
|
||||
|
||||
import type { DisplayPointResolution, MultiDisplaySnapshot } from '../display'
|
||||
import type {
|
||||
ActionInvocation,
|
||||
ComputerUseConfig,
|
||||
DesktopExecutor,
|
||||
DisplayInfo,
|
||||
ForegroundContext,
|
||||
PolicyDecision,
|
||||
ScreenshotArtifact,
|
||||
@@ -15,6 +17,7 @@ import type { ComputerUseServerRuntime } from './runtime'
|
||||
import { normalizeConfiguredAppAction } from '../app-aliases'
|
||||
import { decideBrowserTypeAction } from '../browser-action-router'
|
||||
import { isBrowserDomActionSupported } from '../browser-dom/capabilities'
|
||||
import { resolveDisplayPoint } from '../display'
|
||||
import { evaluateActionPolicy } from '../policy'
|
||||
import { getRuntimePreflight } from '../preflight'
|
||||
import { buildCoordinateSpaceInfo } from '../runtime-probes'
|
||||
@@ -119,6 +122,91 @@ function toTerminalStateContent(state: TerminalState) {
|
||||
}
|
||||
}
|
||||
|
||||
function displaySnapshotFromDisplayInfo(displayInfo: DisplayInfo): MultiDisplaySnapshot | undefined {
|
||||
if (!displayInfo.displays?.length) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return {
|
||||
displays: displayInfo.displays.map(display => ({
|
||||
displayId: display.displayId,
|
||||
isMain: display.isMain,
|
||||
isBuiltIn: display.isBuiltIn,
|
||||
bounds: display.bounds,
|
||||
visibleBounds: display.visibleBounds,
|
||||
scaleFactor: display.scaleFactor,
|
||||
pixelWidth: display.pixelWidth,
|
||||
pixelHeight: display.pixelHeight,
|
||||
})),
|
||||
combinedBounds: displayInfo.combinedBounds ?? {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
},
|
||||
capturedAt: displayInfo.capturedAt ?? new Date(0).toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
function getCoordinateMutationTarget(action: ActionInvocation): { x: number, y: number } | undefined {
|
||||
switch (action.kind) {
|
||||
case 'click':
|
||||
return { x: action.input.x, y: action.input.y }
|
||||
case 'type_text':
|
||||
if (typeof action.input.x === 'number' && typeof action.input.y === 'number') {
|
||||
return { x: action.input.x, y: action.input.y }
|
||||
}
|
||||
return undefined
|
||||
case 'scroll':
|
||||
if (typeof action.input.x === 'number' && typeof action.input.y === 'number') {
|
||||
return { x: action.input.x, y: action.input.y }
|
||||
}
|
||||
return undefined
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function toStructuredDisplayPoint(resolution: DisplayPointResolution) {
|
||||
return {
|
||||
coordinateSpace: 'global-logical',
|
||||
global: resolution.global,
|
||||
displayId: resolution.display.displayId,
|
||||
displayBounds: resolution.display.bounds,
|
||||
local: resolution.local,
|
||||
backingPixel: resolution.backingPixel,
|
||||
scaleFactor: resolution.display.scaleFactor,
|
||||
}
|
||||
}
|
||||
|
||||
function resolveActionDisplayPoint(action: ActionInvocation, displayInfo: DisplayInfo) {
|
||||
const target = getCoordinateMutationTarget(action)
|
||||
const snapshot = displaySnapshotFromDisplayInfo(displayInfo)
|
||||
|
||||
if (!target || !snapshot) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const resolution = resolveDisplayPoint(snapshot, target.x, target.y)
|
||||
if (!resolution) {
|
||||
const combined = displayInfo.combinedBounds
|
||||
return {
|
||||
status: 'outside' as const,
|
||||
target,
|
||||
reason: combined
|
||||
? `target point (${target.x}, ${target.y}) is outside connected display bounds ${combined.width}x${combined.height} @ (${combined.x},${combined.y})`
|
||||
: `target point (${target.x}, ${target.y}) is outside connected display bounds`,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'ok' as const,
|
||||
target,
|
||||
resolution,
|
||||
structured: toStructuredDisplayPoint(resolution),
|
||||
}
|
||||
}
|
||||
|
||||
function getPolicyEvaluationContext(params: {
|
||||
action: ActionInvocation
|
||||
actualContext: ForegroundContext
|
||||
@@ -255,6 +343,33 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
|
||||
return buildDeniedResponse(decision, context, executionTarget)
|
||||
}
|
||||
|
||||
const actionDisplayPoint = resolveActionDisplayPoint(normalizedAction, displayInfo)
|
||||
if (actionDisplayPoint?.status === 'outside') {
|
||||
const deniedDecision = buildDeniedDecision({
|
||||
decision,
|
||||
issues: [actionDisplayPoint.reason],
|
||||
})
|
||||
|
||||
await runtime.session.record({
|
||||
event: 'denied',
|
||||
toolName,
|
||||
action: normalizedAction,
|
||||
context,
|
||||
policy: deniedDecision,
|
||||
result: {
|
||||
executionTarget,
|
||||
displayInfo,
|
||||
coordinateSpace: preflight.coordinateSpace,
|
||||
targetPoint: actionDisplayPoint.target,
|
||||
},
|
||||
})
|
||||
|
||||
return buildDeniedResponse(deniedDecision, context, executionTarget)
|
||||
}
|
||||
const structuredDisplayPoint = actionDisplayPoint?.status === 'ok'
|
||||
? actionDisplayPoint.structured
|
||||
: undefined
|
||||
|
||||
if (decision.requiresApproval && !options.skipApprovalQueue) {
|
||||
const pending = runtime.session.createPendingAction({
|
||||
toolName,
|
||||
@@ -380,6 +495,7 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
|
||||
backendResult = {
|
||||
...result,
|
||||
pointerTrace,
|
||||
displayPoint: structuredDisplayPoint,
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -406,6 +522,7 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
|
||||
})
|
||||
runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y })
|
||||
backendResult.focusPointerTrace = pointerTrace
|
||||
backendResult.focusDisplayPoint = structuredDisplayPoint
|
||||
}
|
||||
catch (clickError) {
|
||||
const msg = clickError instanceof Error ? clickError.message : String(clickError)
|
||||
@@ -479,7 +596,10 @@ export function createExecuteAction(runtime: ComputerUseServerRuntime): ExecuteA
|
||||
if (typeof normalizedAction.input.x === 'number' && typeof normalizedAction.input.y === 'number') {
|
||||
runtime.session.setPointerPosition({ x: normalizedAction.input.x, y: normalizedAction.input.y })
|
||||
}
|
||||
backendResult = { ...result }
|
||||
backendResult = {
|
||||
...result,
|
||||
displayPoint: structuredDisplayPoint,
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'wait': {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
|
||||
|
||||
import type { ComputerUseServerRuntime } from './runtime'
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { RunStateManager } from '../state'
|
||||
import { createTestConfig } from '../test-fixtures'
|
||||
import { registerComputerUseTools } from './register-tools'
|
||||
|
||||
type SchemaShape = Record<string, { description?: string }>
|
||||
|
||||
function createMockServer() {
|
||||
const schemas = new Map<string, SchemaShape>()
|
||||
|
||||
return {
|
||||
server: {
|
||||
tool(name: string, schemaOrSummary: unknown, schemaOrHandler?: unknown) {
|
||||
const schema = typeof schemaOrSummary === 'string'
|
||||
? schemaOrHandler
|
||||
: schemaOrSummary
|
||||
|
||||
schemas.set(name, schema as SchemaShape)
|
||||
|
||||
return { disable: vi.fn() }
|
||||
},
|
||||
} as unknown as McpServer,
|
||||
schemas,
|
||||
}
|
||||
}
|
||||
|
||||
function createRegistrationRuntime() {
|
||||
return {
|
||||
config: createTestConfig(),
|
||||
stateManager: new RunStateManager(),
|
||||
session: {},
|
||||
executor: {},
|
||||
terminalRunner: {},
|
||||
browserDomBridge: {},
|
||||
cdpBridgeManager: {},
|
||||
chromeSessionManager: {},
|
||||
desktopSessionController: {},
|
||||
taskMemory: {},
|
||||
} as unknown as ComputerUseServerRuntime
|
||||
}
|
||||
|
||||
describe('registerComputerUseTools coordinate contract', () => {
|
||||
it('documents mutating desktop target coordinates as global logical screen coordinates', () => {
|
||||
const { server, schemas } = createMockServer()
|
||||
|
||||
registerComputerUseTools({
|
||||
server,
|
||||
runtime: createRegistrationRuntime(),
|
||||
executeAction: vi.fn(),
|
||||
enableTestTools: false,
|
||||
})
|
||||
|
||||
expect(schemas.get('desktop_click')?.x.description).toBe('Global logical screen X coordinate, not Retina backing pixels')
|
||||
expect(schemas.get('desktop_click')?.y.description).toBe('Global logical screen Y coordinate, not Retina backing pixels')
|
||||
expect(schemas.get('desktop_type_text')?.x.description).toBe('Optional global logical screen X coordinate to click before typing')
|
||||
expect(schemas.get('desktop_type_text')?.y.description).toBe('Optional global logical screen Y coordinate to click before typing')
|
||||
expect(schemas.get('desktop_scroll')?.x.description).toBe('Optional global logical screen X coordinate to move to before scrolling')
|
||||
expect(schemas.get('desktop_scroll')?.y.description).toBe('Optional global logical screen Y coordinate to move to before scrolling')
|
||||
})
|
||||
})
|
||||
@@ -242,8 +242,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
|
||||
server.tool(
|
||||
'desktop_click',
|
||||
{
|
||||
x: z.number().describe('Absolute screen X coordinate in pixels'),
|
||||
y: z.number().describe('Absolute screen Y coordinate in pixels'),
|
||||
x: z.number().describe('Global logical screen X coordinate, not Retina backing pixels'),
|
||||
y: z.number().describe('Global logical screen Y coordinate, not Retina backing pixels'),
|
||||
button: z.enum(['left', 'right', 'middle']).optional().describe('Mouse button, default left'),
|
||||
clickCount: z.number().int().min(1).max(2).optional().describe('Number of clicks, default 1'),
|
||||
captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'),
|
||||
@@ -255,8 +255,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
|
||||
'desktop_type_text',
|
||||
{
|
||||
text: z.string().min(1).describe('Text to type into the focused UI element'),
|
||||
x: z.number().optional().describe('Optional X coordinate to click before typing'),
|
||||
y: z.number().optional().describe('Optional Y coordinate to click before typing'),
|
||||
x: z.number().optional().describe('Optional global logical screen X coordinate to click before typing'),
|
||||
y: z.number().optional().describe('Optional global logical screen Y coordinate to click before typing'),
|
||||
pressEnter: z.boolean().optional().describe('Whether to press Enter after typing'),
|
||||
captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'),
|
||||
},
|
||||
@@ -275,8 +275,8 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
|
||||
server.tool(
|
||||
'desktop_scroll',
|
||||
{
|
||||
x: z.number().optional().describe('Optional X coordinate to move to before scrolling'),
|
||||
y: z.number().optional().describe('Optional Y coordinate to move to before scrolling'),
|
||||
x: z.number().optional().describe('Optional global logical screen X coordinate to move to before scrolling'),
|
||||
y: z.number().optional().describe('Optional global logical screen Y coordinate to move to before scrolling'),
|
||||
deltaX: z.number().optional().describe('Horizontal scroll delta in pixels'),
|
||||
deltaY: z.number().describe('Vertical scroll delta in pixels'),
|
||||
captureAfter: z.boolean().optional().describe('Whether to return a fresh screenshot after the action'),
|
||||
|
||||
@@ -117,7 +117,7 @@ export const desktopDescriptors: ToolDescriptor[] = [
|
||||
{
|
||||
canonicalName: 'desktop_click',
|
||||
displayName: 'Desktop Click',
|
||||
summary: 'Click at screen coordinates. Supports left/right/middle buttons and click count for double-click.',
|
||||
summary: 'Click at global logical screen coordinates, not Retina backing pixels. Supports left/right/middle buttons and click count for double-click.',
|
||||
lane: 'desktop',
|
||||
kind: 'write',
|
||||
readOnly: false,
|
||||
@@ -130,7 +130,7 @@ export const desktopDescriptors: ToolDescriptor[] = [
|
||||
{
|
||||
canonicalName: 'desktop_type_text',
|
||||
displayName: 'Desktop Type Text',
|
||||
summary: 'Type text into the focused element. Optionally click at coordinates first. Long text (>160 chars) requires approval.',
|
||||
summary: 'Type text into the focused element. Optionally click at global logical screen coordinates first. Long text (>160 chars) requires approval.',
|
||||
lane: 'desktop',
|
||||
kind: 'write',
|
||||
readOnly: false,
|
||||
@@ -156,7 +156,7 @@ export const desktopDescriptors: ToolDescriptor[] = [
|
||||
{
|
||||
canonicalName: 'desktop_scroll',
|
||||
displayName: 'Desktop Scroll',
|
||||
summary: 'Scroll the screen at optional coordinates. Supports both vertical and horizontal scrolling.',
|
||||
summary: 'Scroll the screen at optional global logical screen coordinates. Supports both vertical and horizontal scrolling.',
|
||||
lane: 'desktop',
|
||||
kind: 'write',
|
||||
readOnly: false,
|
||||
|
||||
@@ -296,6 +296,15 @@ describe('toolDescriptorRegistry', () => {
|
||||
expect(desc.readOnly).toBe(true)
|
||||
expect(desc.public).toBe(true)
|
||||
})
|
||||
|
||||
it('documents direct desktop coordinate tools as global logical coordinates', () => {
|
||||
const registry = createPopulatedRegistry()
|
||||
|
||||
expect(registry.get('desktop_click').summary).toContain('global logical screen coordinates')
|
||||
expect(registry.get('desktop_click').summary).toContain('not Retina backing pixels')
|
||||
expect(registry.get('desktop_type_text').summary).toContain('global logical screen coordinates')
|
||||
expect(registry.get('desktop_scroll').summary).toContain('global logical screen coordinates')
|
||||
})
|
||||
})
|
||||
|
||||
describe('desktop grounding tool enablement', () => {
|
||||
|
||||
@@ -191,7 +191,9 @@ export interface PointerTracePoint {
|
||||
}
|
||||
|
||||
export interface ClickActionInput {
|
||||
/** Global logical screen coordinate, not Retina backing pixels. */
|
||||
x: number
|
||||
/** Global logical screen coordinate, not Retina backing pixels. */
|
||||
y: number
|
||||
button?: MouseButton
|
||||
clickCount?: number
|
||||
@@ -200,7 +202,9 @@ export interface ClickActionInput {
|
||||
|
||||
export interface TypeTextActionInput {
|
||||
text: string
|
||||
/** Optional global logical screen coordinate to focus before typing. */
|
||||
x?: number
|
||||
/** Optional global logical screen coordinate to focus before typing. */
|
||||
y?: number
|
||||
pressEnter?: boolean
|
||||
captureAfter?: boolean
|
||||
@@ -212,7 +216,9 @@ export interface PressKeysActionInput {
|
||||
}
|
||||
|
||||
export interface ScrollActionInput {
|
||||
/** Optional global logical screen coordinate to move to before scrolling. */
|
||||
x?: number
|
||||
/** Optional global logical screen coordinate to move to before scrolling. */
|
||||
y?: number
|
||||
deltaX?: number
|
||||
deltaY: number
|
||||
|
||||
Reference in New Issue
Block a user