From 8239e290bf4c44e63f69d4614082f7376435bde1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A2=93=E6=81=92?= <160735726+3361559784@users.noreply.github.com> Date: Mon, 4 May 2026 06:38:05 +0800 Subject: [PATCH] fix(stage-tamagotchi): add macOS cursor restore discipline (#1754) --- .../src/executors/macos-local.test.ts | 41 ++++++---- .../src/executors/macos-local.ts | 78 +++++++++++++------ 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/services/computer-use-mcp/src/executors/macos-local.test.ts b/services/computer-use-mcp/src/executors/macos-local.test.ts index 9dfa0a02c..2a8d304d4 100644 --- a/services/computer-use-mcp/src/executors/macos-local.test.ts +++ b/services/computer-use-mcp/src/executors/macos-local.test.ts @@ -1,23 +1,32 @@ import { describe, expect, it } from 'vitest' -import { moveAndClickScript } from './macos-local' +import { + buildMacOSMoveAndClickScript, + buildMacOSPressKeysScript, + buildMacOSScrollScript, + buildMacOSTypeTextScript, +} from './macos-local' -describe('moveAndClickScript', () => { - it('saves and restores the real macOS cursor around CGEvent clicks', () => { - const source = moveAndClickScript() +describe('macOS local Swift cursor state contract', () => { + it('leaves desktop_click cursor at the action target for pointer-relative follow-up actions', () => { + const script = buildMacOSMoveAndClickScript() - const saveCursorIndex = source.indexOf('let originalCursorLocation = CGEvent(source: nil)?.location') - const restoreCursorIndex = source.indexOf('CGWarpMouseCursorPosition(savedCursorLocation)') - const moveTraceIndex = source.indexOf('for point in trace') - const clickIndex = source.indexOf('down.post(tap: .cghidEventTap)') + expect(script).not.toContain('let originalCursorLocation = CGEvent(source: nil)?.location') + expect(script).not.toContain('CGWarpMouseCursorPosition') + expect(script).toContain('for point in trace') + }) - expect(saveCursorIndex).toBeGreaterThanOrEqual(0) - expect(restoreCursorIndex).toBeGreaterThanOrEqual(0) - expect(moveTraceIndex).toBeGreaterThanOrEqual(0) - expect(clickIndex).toBeGreaterThanOrEqual(0) - expect(saveCursorIndex).toBeLessThan(moveTraceIndex) - expect(restoreCursorIndex).toBeLessThan(moveTraceIndex) - expect(restoreCursorIndex).toBeLessThan(clickIndex) - expect(source).toContain('defer {') + it('leaves coordinate-based desktop_scroll cursor at the target for pointer-relative follow-up actions', () => { + const script = buildMacOSScrollScript() + + expect(script).not.toContain('let shouldRestoreCursor = x != nil && y != nil') + expect(script).not.toContain('CGWarpMouseCursorPosition') + expect(script).toContain('if let x, let y {') + expect(script).toContain('scrollEvent.post(tap: .cghidEventTap)') + }) + + it('does not warp the cursor for keyboard-only text and key-chord scripts', () => { + expect(buildMacOSTypeTextScript()).not.toContain('CGWarpMouseCursorPosition') + expect(buildMacOSPressKeysScript(36, '[]')).not.toContain('CGWarpMouseCursorPosition') }) }) diff --git a/services/computer-use-mcp/src/executors/macos-local.ts b/services/computer-use-mcp/src/executors/macos-local.ts index fb0d95054..70eb39e70 100644 --- a/services/computer-use-mcp/src/executors/macos-local.ts +++ b/services/computer-use-mcp/src/executors/macos-local.ts @@ -211,20 +211,20 @@ print(String(data: data, encoding: .utf8)!) } /** - * Creates Swift source that posts a local macOS click while restoring the user's cursor. + * Builds the Swift script used for local macOS pointer movement and clicks. * * Use when: - * - The macOS executor needs to perform a pointer-based click through Quartz. - * - Tests need to inspect the generated Swift source without moving the real cursor. + * - Executing `desktop_click` on the local macOS host + * - Verifying the Quartz cursor state contract in unit tests * * Expects: - * - `COMPUTER_USE_SWIFT_STDIN` contains `pointerTrace`, `button`, and `clickCount`. - * - The caller already verified that the host platform is macOS. + * - JSON input is provided through `COMPUTER_USE_SWIFT_STDIN` + * - Pointer coordinates are already global logical macOS screen coordinates * * Returns: - * - Swift source that saves the real cursor location before CGEvent movement and restores it with `CGWarpMouseCursorPosition`. + * - A Swift script that leaves the real cursor at the clicked target for follow-up pointer-relative actions */ -export function moveAndClickScript() { +export function buildMacOSMoveAndClickScript(): string { return String.raw` import CoreGraphics import Foundation @@ -262,16 +262,6 @@ let buttonRaw = input["button"] as? Int ?? 0 let clickCount = input["clickCount"] as? Int ?? 1 let button = mouseButton(buttonRaw) -// CGEvent mouse posts move the real macOS cursor. Save and restore it so the -// overlay ghost pointer can visualize agent intent without leaving the user's -// actual cursor at the agent click target. -let originalCursorLocation = CGEvent(source: nil)?.location -defer { - if let savedCursorLocation = originalCursorLocation { - CGWarpMouseCursorPosition(savedCursorLocation) - } -} - for point in trace { let x = point["x"] as? Double ?? 0 let y = point["y"] as? Double ?? 0 @@ -304,7 +294,20 @@ print("{}") ` } -function typeTextScript() { +/** + * Builds the Swift script used for local macOS keyboard text injection. + * + * Use when: + * - Executing `desktop_type_text` after any optional focus click has completed + * - Verifying keyboard-only scripts do not warp the user's cursor + * + * Expects: + * - JSON input is provided through `COMPUTER_USE_SWIFT_STDIN` + * + * Returns: + * - A Swift script that posts keyboard events without moving the mouse cursor + */ +export function buildMacOSTypeTextScript(): string { return String.raw` import CoreGraphics import Foundation @@ -359,7 +362,21 @@ print("{}") ` } -function pressKeysScript(mainKeyCode: number, modifierMaskExpr: string) { +/** + * Builds the Swift script used for local macOS key chord injection. + * + * Use when: + * - Executing `desktop_press_keys` + * - Verifying keyboard-only scripts do not warp the user's cursor + * + * Expects: + * - `mainKeyCode` is a supported macOS virtual key code + * - `modifierMaskExpr` is a Swift `CGEventFlags` expression + * + * Returns: + * - A Swift script that posts keyboard events without moving the mouse cursor + */ +export function buildMacOSPressKeysScript(mainKeyCode: number, modifierMaskExpr: string): string { return String.raw` import CoreGraphics import Foundation @@ -379,7 +396,20 @@ print("{}") ` } -function scrollScript() { +/** + * Builds the Swift script used for local macOS scroll events. + * + * Use when: + * - Executing `desktop_scroll` on the local macOS host + * - Verifying cursor restore only when scroll input includes a target point + * + * Expects: + * - Optional `x`/`y` coordinates are global logical macOS screen coordinates + * + * Returns: + * - A Swift script that leaves coordinate-based scroll cursor state aligned with follow-up pointer-relative actions + */ +export function buildMacOSScrollScript(): string { return String.raw` import CoreGraphics import Foundation @@ -523,7 +553,7 @@ export function createMacOSLocalExecutor(config: ComputerUseConfig): DesktopExec }, click: async (input: ClickActionInput & { pointerTrace: PointerTracePoint[] }) => { await ensureMacOS() - await runMacOsJsonScript>(config, moveAndClickScript(), { + await runMacOsJsonScript>(config, buildMacOSMoveAndClickScript(), { pointerTrace: input.pointerTrace, button: buttonNames[input.button || 'left'], clickCount: input.clickCount ?? 1, @@ -535,7 +565,7 @@ export function createMacOSLocalExecutor(config: ComputerUseConfig): DesktopExec }, typeText: async (input: TypeTextActionInput) => { await ensureMacOS() - await runMacOsJsonScript>(config, typeTextScript(), { + await runMacOsJsonScript>(config, buildMacOSTypeTextScript(), { text: input.text, pressEnter: input.pressEnter ?? false, }) @@ -563,12 +593,12 @@ export function createMacOSLocalExecutor(config: ComputerUseConfig): DesktopExec }).join(' | ') : '[]' - await runMacOsJsonScript>(config, pressKeysScript(keyCode, modifierMaskExpr), {}) + await runMacOsJsonScript>(config, buildMacOSPressKeysScript(keyCode, modifierMaskExpr), {}) return result([`pressed keys ${normalized.join('+')}`], executionTarget) }, scroll: async (input: ScrollActionInput) => { await ensureMacOS() - await runMacOsJsonScript>(config, scrollScript(), { + await runMacOsJsonScript>(config, buildMacOSScrollScript(), { x: input.x, y: input.y, deltaX: input.deltaX ?? 0,