feat(computer-use-mcp): add terminal screen heuristics (#1738)

2026-05-13 11:56:09 +08:00
parent cad10ca090
commit f81e116022
5 changed files with 345 additions and 13 deletions
@@ -190,4 +190,72 @@ describe('registerPtyTools', () => {
      }),
    ])
  })
+
+  it('returns pagination nudges from pty_read_screen heuristics', async () => {
+    runtime.stateManager.registerPtySession({
+      id: 'pty_1',
+      alive: true,
+      rows: 24,
+      cols: 80,
+      pid: 9001,
+      cwd: '/tmp/project',
+    })
+    vi.mocked(readPtyScreen).mockReturnValue({
+      id: 'pty_1',
+      alive: true,
+      rows: 24,
+      cols: 80,
+      screenContent: 'line 1\nline 2\n--More--\n',
+      pid: 9001,
+    })
+    const { server, invoke } = createMockServer()
+
+    registerPtyTools({ server, runtime })
+
+    const result = await invoke('pty_read_screen', {
+      sessionId: 'pty_1',
+    })
+
+    expect((result.structuredContent as Record<string, any>).suggestedInteraction).toBe('press_space')
+    expect(result.content).toEqual(expect.arrayContaining([
+      expect.objectContaining({
+        text: expect.stringContaining('You may need to press Space'),
+      }),
+    ]))
+  })
+
+  it('records observed cwd from the last non-empty prompt line without mutating creation cwd', async () => {
+    runtime.stateManager.registerPtySession({
+      id: 'pty_1',
+      alive: true,
+      rows: 24,
+      cols: 80,
+      pid: 9001,
+      cwd: '/tmp/project',
+    })
+    vi.mocked(readPtyScreen).mockReturnValue({
+      id: 'pty_1',
+      alive: true,
+      rows: 24,
+      cols: 80,
+      screenContent: 'cd /tmp/next\n\u001B[32malice@wonderland\u001B[0m:\u001B[34m/tmp/next\u001B[0m$ \n',
+      pid: 9001,
+    })
+    const { server, invoke } = createMockServer()
+
+    registerPtyTools({ server, runtime })
+
+    const result = await invoke('pty_read_screen', {
+      sessionId: 'pty_1',
+    })
+
+    expect((result.structuredContent as Record<string, any>).observedCwd).toBe('/tmp/next')
+    expect(runtime.stateManager.getPtySessions()).toEqual([
+      expect.objectContaining({
+        id: 'pty_1',
+        cwd: '/tmp/project',
+        observedCwd: '/tmp/next',
+      }),
+    ])
+  })
 })
@@ -24,6 +24,7 @@ import {
 } from '../terminal/pty-runner'
 import { textContent } from './content'
 import { buildApprovalResponse } from './responses'
+import { detectPagination, extractCwdFromPrompt } from './terminal-heuristics'

 export interface RegisterPtyToolsOptions {
  server: McpServer
@@ -430,24 +431,53 @@ export function registerPtyTools({ server, runtime }: RegisterPtyToolsOptions) {
          alive: session.alive,
        })

-        return {
-          content: [textContent(session.screenContent || '(empty)')],
-          structuredContent: {
-            status: 'ok',
-            session: {
-              id: session.id,
-              alive: session.alive,
-              pid: session.pid,
-              rows: session.rows,
-              cols: session.cols,
-            },
-            sessionId: session.id,
+        const structuredContent: Record<string, unknown> = {
+          status: 'ok',
+          session: {
+            id: session.id,
            alive: session.alive,
+            pid: session.pid,
            rows: session.rows,
            cols: session.cols,
-            screenContent: session.screenContent,
          },
+          sessionId: session.id,
+          alive: session.alive,
+          rows: session.rows,
+          cols: session.cols,
+          screenContent: session.screenContent,
        }
+
+        const response: CallToolResult = {
+          content: [textContent(session.screenContent || '(empty)')],
+          structuredContent,
+        }
+
+        // --- Hygiene Heuristics ---
+        const content = session.screenContent || ''
+        const lines = content.split('\n')
+        let lastLine = ''
+        for (let index = lines.length - 1; index >= 0; index -= 1) {
+          if (lines[index].trim().length > 0) {
+            lastLine = lines[index]
+            break
+          }
+        }
+
+        // 1. Pagination Nudge
+        const pagination = detectPagination(content)
+        if (pagination) {
+          response.content.push(textContent(`\n[NUDGE] ${pagination.reason}. You may need to press ${pagination.suggestedAction === 'press_space' ? 'Space' : 'q'}.`))
+          structuredContent.suggestedInteraction = pagination.suggestedAction
+        }
+
+        // 2. Best-effort CWD Recovery
+        const extractedCwd = extractCwdFromPrompt(lastLine)
+        if (extractedCwd) {
+          runtime.stateManager.updatePtySessionObservedCwd(sessionId, extractedCwd)
+          structuredContent.observedCwd = extractedCwd
+        }
+
+        return response
      }
      catch (error) {
        return {
@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest'
+
+import { detectPagination, extractCwdFromPrompt } from './terminal-heuristics'
+
+describe('terminal-heuristics', () => {
+  describe('detectPagination', () => {
+    it('detects --More-- prompts', () => {
+      const content = 'line 1\nline 2\n--More--'
+      const result = detectPagination(content)
+      expect(result).toBeDefined()
+      expect(result?.suggestedAction).toBe('press_space')
+    })
+
+    it('detects ANSI-styled --More-- prompts', () => {
+      const content = 'line 1\nline 2\n\u001B[7m--More--\u001B[0m'
+      const result = detectPagination(content)
+      expect(result).toBeDefined()
+      expect(result?.suggestedAction).toBe('press_space')
+    })
+
+    it('detects (END) markers', () => {
+      const content = 'some log content\n\u001B[7m(END)\u001B[0m\n'
+      const result = detectPagination(content)
+      expect(result).toBeDefined()
+      expect(result?.suggestedAction).toBe('press_q')
+    })
+
+    it('ignores ordinary output that merely mentions (END)', () => {
+      const content = 'job status: reached (END) marker in parser output'
+      const result = detectPagination(content)
+      expect(result).toBeUndefined()
+    })
+
+    it('detects trailing colon only with pager context', () => {
+      const content = 'Manual page printf(1) line 1\n:'
+      const result = detectPagination(content)
+      expect(result).toBeDefined()
+      expect(result?.suggestedAction).toBe('press_q')
+    })
+
+    it('ignores bare trailing colon without pager context', () => {
+      const content = 'ordinary command output\n:'
+      const result = detectPagination(content)
+      expect(result).toBeUndefined()
+    })
+
+    it('returns undefined for normal output', () => {
+      const content = 'user@host:~$ ls -l\ntotal 0'
+      const result = detectPagination(content)
+      expect(result).toBeUndefined()
+    })
+  })
+
+  describe('extractCwdFromPrompt', () => {
+    it('extracts path from bash/zsh default style', () => {
+      expect(extractCwdFromPrompt('alice@wonderland:~/rabbit-hole$ ')).toBe('~/rabbit-hole')
+      expect(extractCwdFromPrompt('root@localhost:/etc# ')).toBe('/etc')
+    })
+
+    it('extracts path from ANSI-styled prompts', () => {
+      expect(extractCwdFromPrompt('\u001B[32malice@wonderland\u001B[0m:\u001B[34m~/rabbit-hole\u001B[0m$ ')).toBe('~/rabbit-hole')
+    })
+
+    it('extracts path from CentOS/brackets style', () => {
+      expect(extractCwdFromPrompt('[bob@server /var/log]$ ')).toBe('/var/log')
+      expect(extractCwdFromPrompt('[alice@home ~]# ')).toBe('~')
+    })
+
+    it('returns undefined for non-prompt lines', () => {
+      expect(extractCwdFromPrompt('total 123')).toBeUndefined()
+      expect(extractCwdFromPrompt('drwxr-xr-x  2 root  root  4096 Apr  9 18:00 .')).toBeUndefined()
+    })
+  })
+})
@@ -0,0 +1,149 @@
+/**
+ * Heuristics for interpreting terminal screen content.
+ * These are purely observational and do not modify the state of the PTY.
+ */
+
+export interface TerminalHeuristicsResult {
+  pagination?: {
+    suggestedAction: 'press_space' | 'press_q'
+    reason: string
+  }
+  extractedCwd?: string
+}
+
+function stripAnsiEscapeCodes(value: string): string {
+  let stripped = ''
+
+  for (let index = 0; index < value.length;) {
+    if (value[index] !== '\u001B') {
+      stripped += value[index]
+      index += 1
+      continue
+    }
+
+    index += 1
+    const marker = value[index]
+
+    if (marker === '[') {
+      index += 1
+      while (index < value.length) {
+        const code = value.charCodeAt(index)
+        index += 1
+        if (code >= 0x40 && code <= 0x7E)
+          break
+      }
+      continue
+    }
+
+    if (marker === ']') {
+      index += 1
+      while (index < value.length) {
+        if (value.charCodeAt(index) === 0x07) {
+          index += 1
+          break
+        }
+        if (value[index] === '\u001B' && value[index + 1] === '\\') {
+          index += 2
+          break
+        }
+        index += 1
+      }
+      continue
+    }
+
+    if (marker)
+      index += 1
+  }
+
+  return stripped
+}
+
+function hasPagerContext(lines: string[]): boolean {
+  const recentLines = lines.slice(Math.max(0, lines.length - 6), -1)
+
+  return recentLines.some((line) => {
+    return line.includes('Manual page')
+      || line.includes('press h for help')
+      || /^lines?\s+\d+[-,]\d+/i.test(line)
+      || /^\d+%$/.test(line)
+  })
+}
+
+/**
+ * Detects common pagination markers (more, less, etc.) in terminal output.
+ */
+export function detectPagination(screenContent: string): TerminalHeuristicsResult['pagination'] | undefined {
+  if (!screenContent)
+    return undefined
+
+  const lines = stripAnsiEscapeCodes(screenContent)
+    .split('\n')
+    .map(line => line.trim())
+    .filter(line => line.length > 0)
+
+  if (lines.length === 0)
+    return undefined
+
+  const lastLine = lines[lines.length - 1]
+  const secondLastLine = lines.length > 1 ? lines[lines.length - 2].trim() : ''
+
+  // Common pagination patterns
+  if (lastLine.includes('--More--')) {
+    return { suggestedAction: 'press_space', reason: 'Pagination detected (--More--)' }
+  }
+
+  if (lastLine === '(END)') {
+    return { suggestedAction: 'press_q', reason: 'End of output or pager prompt detected' }
+  }
+
+  if (lastLine === ':' && hasPagerContext(lines)) {
+    return { suggestedAction: 'press_q', reason: 'Pager prompt detected' }
+  }
+
+  // Sometimes help or man pages end with a specific manual prompt
+  if (secondLastLine.includes('Manual page') && lastLine.includes('line 1')) {
+    return { suggestedAction: 'press_q', reason: 'Man page detected' }
+  }
+
+  return undefined
+}
+
+/**
+ * Best-effort extraction of CWD from a terminal prompt.
+ * Supported patterns:
+ * - user@host:path$
+ * - [user@host path]$
+ * - path >
+ */
+export function extractCwdFromPrompt(line: string): string | undefined {
+  const cleanLine = stripAnsiEscapeCodes(line).trim()
+  if (!cleanLine || cleanLine.length > 200)
+    return undefined
+
+  // Pattern 1: user@host:path$ (typical bash/zsh default)
+  const pattern1 = /[\w.-]+@[\w.-]+:([^$#\s>]+)\s*[$#]\s*$/
+  const match1 = cleanLine.match(pattern1)
+  if (match1)
+    return match1[1]
+
+  // Pattern 2: [user@host path]$ (CentOS/RHEL)
+  if (cleanLine.startsWith('[')) {
+    const closeBracketIndex = cleanLine.lastIndexOf(']')
+    if (closeBracketIndex > 0) {
+      const suffix = cleanLine.slice(closeBracketIndex + 1).trim()
+      const promptBody = cleanLine.slice(1, closeBracketIndex)
+      const firstSpaceIndex = promptBody.indexOf(' ')
+      if ((suffix === '$' || suffix === '#') && firstSpaceIndex > 0) {
+        return promptBody.slice(firstSpaceIndex + 1)
+      }
+    }
+  }
+
+  // Pattern 3: Simple path > (generic)
+  const pattern3 = /^(\/(?:[\w.-]+\/)*[\w.-]+)\s*>\s*$/
+  const match3 = cleanLine.match(pattern3)
+  if (match3)
+    return match3[1]
+
+  return undefined
+}
@@ -59,6 +59,8 @@ export interface PtySessionState {
  pid: number
  /** Working directory at creation time. */
  cwd?: string
+  /** Last cwd observed from terminal prompt heuristics. */
+  observedCwd?: string
  /** Stable workflow step id that created this session (if any). */
  boundStepId?: string
  /**
@@ -538,6 +540,15 @@ export class RunStateManager {
    }
  }

+  /** Update the last observed cwd of a PTY session from terminal prompt heuristics. */
+  updatePtySessionObservedCwd(sessionId: string, cwd: string): void {
+    const entry = this.state.ptySessions.find(s => s.id === sessionId)
+    if (entry && entry.observedCwd !== cwd) {
+      entry.observedCwd = cwd
+      this.touch()
+    }
+  }
+
  /** Bind a PTY session to a workflow step by stable stepId. */
  bindPtySessionToStepId(sessionId: string, stepId: string): void {
    const entry = this.state.ptySessions.find(s => s.id === sessionId)