From f81e11602291bf10ba776e0819b70b85c916a797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A2=93=E6=81=92?= <160735726+3361559784@users.noreply.github.com> Date: Wed, 13 May 2026 11:56:09 +0800 Subject: [PATCH] feat(computer-use-mcp): add terminal screen heuristics (#1738) --- .../src/server/register-pty.test.ts | 68 ++++++++ .../src/server/register-pty.ts | 56 +++++-- .../src/server/terminal-heuristics.test.ts | 74 +++++++++ .../src/server/terminal-heuristics.ts | 149 ++++++++++++++++++ services/computer-use-mcp/src/state.ts | 11 ++ 5 files changed, 345 insertions(+), 13 deletions(-) create mode 100644 services/computer-use-mcp/src/server/terminal-heuristics.test.ts create mode 100644 services/computer-use-mcp/src/server/terminal-heuristics.ts diff --git a/services/computer-use-mcp/src/server/register-pty.test.ts b/services/computer-use-mcp/src/server/register-pty.test.ts index c69e7c193..47e47aab8 100644 --- a/services/computer-use-mcp/src/server/register-pty.test.ts +++ b/services/computer-use-mcp/src/server/register-pty.test.ts @@ -190,4 +190,72 @@ describe('registerPtyTools', () => { }), ]) }) + + it('returns pagination nudges from pty_read_screen heuristics', async () => { + runtime.stateManager.registerPtySession({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + pid: 9001, + cwd: '/tmp/project', + }) + vi.mocked(readPtyScreen).mockReturnValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: 'line 1\nline 2\n--More--\n', + pid: 9001, + }) + const { server, invoke } = createMockServer() + + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_read_screen', { + sessionId: 'pty_1', + }) + + expect((result.structuredContent as Record).suggestedInteraction).toBe('press_space') + expect(result.content).toEqual(expect.arrayContaining([ + expect.objectContaining({ + text: expect.stringContaining('You may need to press Space'), + }), + ])) + }) + + it('records observed cwd from the last non-empty prompt line without mutating creation cwd', async () => { + runtime.stateManager.registerPtySession({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + pid: 9001, + cwd: '/tmp/project', + }) + vi.mocked(readPtyScreen).mockReturnValue({ + id: 'pty_1', + alive: true, + rows: 24, + cols: 80, + screenContent: 'cd /tmp/next\n\u001B[32malice@wonderland\u001B[0m:\u001B[34m/tmp/next\u001B[0m$ \n', + pid: 9001, + }) + const { server, invoke } = createMockServer() + + registerPtyTools({ server, runtime }) + + const result = await invoke('pty_read_screen', { + sessionId: 'pty_1', + }) + + expect((result.structuredContent as Record).observedCwd).toBe('/tmp/next') + expect(runtime.stateManager.getPtySessions()).toEqual([ + expect.objectContaining({ + id: 'pty_1', + cwd: '/tmp/project', + observedCwd: '/tmp/next', + }), + ]) + }) }) diff --git a/services/computer-use-mcp/src/server/register-pty.ts b/services/computer-use-mcp/src/server/register-pty.ts index 547afbafe..4b59a4906 100644 --- a/services/computer-use-mcp/src/server/register-pty.ts +++ b/services/computer-use-mcp/src/server/register-pty.ts @@ -24,6 +24,7 @@ import { } from '../terminal/pty-runner' import { textContent } from './content' import { buildApprovalResponse } from './responses' +import { detectPagination, extractCwdFromPrompt } from './terminal-heuristics' export interface RegisterPtyToolsOptions { server: McpServer @@ -430,24 +431,53 @@ export function registerPtyTools({ server, runtime }: RegisterPtyToolsOptions) { alive: session.alive, }) - return { - content: [textContent(session.screenContent || '(empty)')], - structuredContent: { - status: 'ok', - session: { - id: session.id, - alive: session.alive, - pid: session.pid, - rows: session.rows, - cols: session.cols, - }, - sessionId: session.id, + const structuredContent: Record = { + status: 'ok', + session: { + id: session.id, alive: session.alive, + pid: session.pid, rows: session.rows, cols: session.cols, - screenContent: session.screenContent, }, + sessionId: session.id, + alive: session.alive, + rows: session.rows, + cols: session.cols, + screenContent: session.screenContent, } + + const response: CallToolResult = { + content: [textContent(session.screenContent || '(empty)')], + structuredContent, + } + + // --- Hygiene Heuristics --- + const content = session.screenContent || '' + const lines = content.split('\n') + let lastLine = '' + for (let index = lines.length - 1; index >= 0; index -= 1) { + if (lines[index].trim().length > 0) { + lastLine = lines[index] + break + } + } + + // 1. Pagination Nudge + const pagination = detectPagination(content) + if (pagination) { + response.content.push(textContent(`\n[NUDGE] ${pagination.reason}. You may need to press ${pagination.suggestedAction === 'press_space' ? 'Space' : 'q'}.`)) + structuredContent.suggestedInteraction = pagination.suggestedAction + } + + // 2. Best-effort CWD Recovery + const extractedCwd = extractCwdFromPrompt(lastLine) + if (extractedCwd) { + runtime.stateManager.updatePtySessionObservedCwd(sessionId, extractedCwd) + structuredContent.observedCwd = extractedCwd + } + + return response } catch (error) { return { diff --git a/services/computer-use-mcp/src/server/terminal-heuristics.test.ts b/services/computer-use-mcp/src/server/terminal-heuristics.test.ts new file mode 100644 index 000000000..f4aeaf45b --- /dev/null +++ b/services/computer-use-mcp/src/server/terminal-heuristics.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest' + +import { detectPagination, extractCwdFromPrompt } from './terminal-heuristics' + +describe('terminal-heuristics', () => { + describe('detectPagination', () => { + it('detects --More-- prompts', () => { + const content = 'line 1\nline 2\n--More--' + const result = detectPagination(content) + expect(result).toBeDefined() + expect(result?.suggestedAction).toBe('press_space') + }) + + it('detects ANSI-styled --More-- prompts', () => { + const content = 'line 1\nline 2\n\u001B[7m--More--\u001B[0m' + const result = detectPagination(content) + expect(result).toBeDefined() + expect(result?.suggestedAction).toBe('press_space') + }) + + it('detects (END) markers', () => { + const content = 'some log content\n\u001B[7m(END)\u001B[0m\n' + const result = detectPagination(content) + expect(result).toBeDefined() + expect(result?.suggestedAction).toBe('press_q') + }) + + it('ignores ordinary output that merely mentions (END)', () => { + const content = 'job status: reached (END) marker in parser output' + const result = detectPagination(content) + expect(result).toBeUndefined() + }) + + it('detects trailing colon only with pager context', () => { + const content = 'Manual page printf(1) line 1\n:' + const result = detectPagination(content) + expect(result).toBeDefined() + expect(result?.suggestedAction).toBe('press_q') + }) + + it('ignores bare trailing colon without pager context', () => { + const content = 'ordinary command output\n:' + const result = detectPagination(content) + expect(result).toBeUndefined() + }) + + it('returns undefined for normal output', () => { + const content = 'user@host:~$ ls -l\ntotal 0' + const result = detectPagination(content) + expect(result).toBeUndefined() + }) + }) + + describe('extractCwdFromPrompt', () => { + it('extracts path from bash/zsh default style', () => { + expect(extractCwdFromPrompt('alice@wonderland:~/rabbit-hole$ ')).toBe('~/rabbit-hole') + expect(extractCwdFromPrompt('root@localhost:/etc# ')).toBe('/etc') + }) + + it('extracts path from ANSI-styled prompts', () => { + expect(extractCwdFromPrompt('\u001B[32malice@wonderland\u001B[0m:\u001B[34m~/rabbit-hole\u001B[0m$ ')).toBe('~/rabbit-hole') + }) + + it('extracts path from CentOS/brackets style', () => { + expect(extractCwdFromPrompt('[bob@server /var/log]$ ')).toBe('/var/log') + expect(extractCwdFromPrompt('[alice@home ~]# ')).toBe('~') + }) + + it('returns undefined for non-prompt lines', () => { + expect(extractCwdFromPrompt('total 123')).toBeUndefined() + expect(extractCwdFromPrompt('drwxr-xr-x 2 root root 4096 Apr 9 18:00 .')).toBeUndefined() + }) + }) +}) diff --git a/services/computer-use-mcp/src/server/terminal-heuristics.ts b/services/computer-use-mcp/src/server/terminal-heuristics.ts new file mode 100644 index 000000000..de6923f1a --- /dev/null +++ b/services/computer-use-mcp/src/server/terminal-heuristics.ts @@ -0,0 +1,149 @@ +/** + * Heuristics for interpreting terminal screen content. + * These are purely observational and do not modify the state of the PTY. + */ + +export interface TerminalHeuristicsResult { + pagination?: { + suggestedAction: 'press_space' | 'press_q' + reason: string + } + extractedCwd?: string +} + +function stripAnsiEscapeCodes(value: string): string { + let stripped = '' + + for (let index = 0; index < value.length;) { + if (value[index] !== '\u001B') { + stripped += value[index] + index += 1 + continue + } + + index += 1 + const marker = value[index] + + if (marker === '[') { + index += 1 + while (index < value.length) { + const code = value.charCodeAt(index) + index += 1 + if (code >= 0x40 && code <= 0x7E) + break + } + continue + } + + if (marker === ']') { + index += 1 + while (index < value.length) { + if (value.charCodeAt(index) === 0x07) { + index += 1 + break + } + if (value[index] === '\u001B' && value[index + 1] === '\\') { + index += 2 + break + } + index += 1 + } + continue + } + + if (marker) + index += 1 + } + + return stripped +} + +function hasPagerContext(lines: string[]): boolean { + const recentLines = lines.slice(Math.max(0, lines.length - 6), -1) + + return recentLines.some((line) => { + return line.includes('Manual page') + || line.includes('press h for help') + || /^lines?\s+\d+[-,]\d+/i.test(line) + || /^\d+%$/.test(line) + }) +} + +/** + * Detects common pagination markers (more, less, etc.) in terminal output. + */ +export function detectPagination(screenContent: string): TerminalHeuristicsResult['pagination'] | undefined { + if (!screenContent) + return undefined + + const lines = stripAnsiEscapeCodes(screenContent) + .split('\n') + .map(line => line.trim()) + .filter(line => line.length > 0) + + if (lines.length === 0) + return undefined + + const lastLine = lines[lines.length - 1] + const secondLastLine = lines.length > 1 ? lines[lines.length - 2].trim() : '' + + // Common pagination patterns + if (lastLine.includes('--More--')) { + return { suggestedAction: 'press_space', reason: 'Pagination detected (--More--)' } + } + + if (lastLine === '(END)') { + return { suggestedAction: 'press_q', reason: 'End of output or pager prompt detected' } + } + + if (lastLine === ':' && hasPagerContext(lines)) { + return { suggestedAction: 'press_q', reason: 'Pager prompt detected' } + } + + // Sometimes help or man pages end with a specific manual prompt + if (secondLastLine.includes('Manual page') && lastLine.includes('line 1')) { + return { suggestedAction: 'press_q', reason: 'Man page detected' } + } + + return undefined +} + +/** + * Best-effort extraction of CWD from a terminal prompt. + * Supported patterns: + * - user@host:path$ + * - [user@host path]$ + * - path > + */ +export function extractCwdFromPrompt(line: string): string | undefined { + const cleanLine = stripAnsiEscapeCodes(line).trim() + if (!cleanLine || cleanLine.length > 200) + return undefined + + // Pattern 1: user@host:path$ (typical bash/zsh default) + const pattern1 = /[\w.-]+@[\w.-]+:([^$#\s>]+)\s*[$#]\s*$/ + const match1 = cleanLine.match(pattern1) + if (match1) + return match1[1] + + // Pattern 2: [user@host path]$ (CentOS/RHEL) + if (cleanLine.startsWith('[')) { + const closeBracketIndex = cleanLine.lastIndexOf(']') + if (closeBracketIndex > 0) { + const suffix = cleanLine.slice(closeBracketIndex + 1).trim() + const promptBody = cleanLine.slice(1, closeBracketIndex) + const firstSpaceIndex = promptBody.indexOf(' ') + if ((suffix === '$' || suffix === '#') && firstSpaceIndex > 0) { + return promptBody.slice(firstSpaceIndex + 1) + } + } + } + + // Pattern 3: Simple path > (generic) + const pattern3 = /^(\/(?:[\w.-]+\/)*[\w.-]+)\s*>\s*$/ + const match3 = cleanLine.match(pattern3) + if (match3) + return match3[1] + + return undefined +} diff --git a/services/computer-use-mcp/src/state.ts b/services/computer-use-mcp/src/state.ts index 97559ed2c..7a194f105 100644 --- a/services/computer-use-mcp/src/state.ts +++ b/services/computer-use-mcp/src/state.ts @@ -59,6 +59,8 @@ export interface PtySessionState { pid: number /** Working directory at creation time. */ cwd?: string + /** Last cwd observed from terminal prompt heuristics. */ + observedCwd?: string /** Stable workflow step id that created this session (if any). */ boundStepId?: string /** @@ -538,6 +540,15 @@ export class RunStateManager { } } + /** Update the last observed cwd of a PTY session from terminal prompt heuristics. */ + updatePtySessionObservedCwd(sessionId: string, cwd: string): void { + const entry = this.state.ptySessions.find(s => s.id === sessionId) + if (entry && entry.observedCwd !== cwd) { + entry.observedCwd = cwd + this.touch() + } + } + /** Bind a PTY session to a workflow step by stable stepId. */ bindPtySessionToStepId(sessionId: string, stepId: string): void { const entry = this.state.ptySessions.find(s => s.id === sessionId)