From 6d7db23f6cfa6ce9796823580edbeb235c744705 Mon Sep 17 00:00:00 2001 From: Rin Date: Wed, 4 Feb 2026 04:08:56 +0800 Subject: [PATCH] feat(minecraft): `expect` guardrails and a working gaze api Add toCoord/cloneVec3 helpers, return structured telemetry from goToPlayer/goToCoordinate with ok/startPos/endPos/movedDistance/distanceToTargetBefore/distanceToTargetAfter fields, track lastPlannerOutcome summary (actionCount/okCount/errorCount/returnValue/logs) in Brain and inject as [SCRIPT] context with truncation, implement expect/expectMoved/expectNear guardrail tools in JavaScriptPlanner to validate action --- .../src/cognitive/action/llm-actions.ts | 55 ++++++++++- .../src/cognitive/conscious/brain.ts | 34 ++++++- .../cognitive/conscious/js-planner.test.ts | 34 +++++++ .../src/cognitive/conscious/js-planner.ts | 95 +++++++++++++++++++ .../conscious/prompts/brain-prompt.ts | 25 +++++ .../src/cognitive/reflex/gaze.test.ts | 51 ++++++++++ .../minecraft/src/cognitive/reflex/gaze.ts | 3 +- 7 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 services/minecraft/src/cognitive/reflex/gaze.test.ts diff --git a/services/minecraft/src/cognitive/action/llm-actions.ts b/services/minecraft/src/cognitive/action/llm-actions.ts index 295f896ad..5bf0ab201 100644 --- a/services/minecraft/src/cognitive/action/llm-actions.ts +++ b/services/minecraft/src/cognitive/action/llm-actions.ts @@ -24,6 +24,14 @@ function formatWearingItem(slot: string, item: string | undefined): string { return item ? `\n${slot}: ${item}` : '' } +function toCoord(pos: { x: number, y: number, z: number }) { + return { x: pos.x, y: pos.y, z: pos.z } +} + +function cloneVec3(pos: { x: number, y: number, z: number }): Vec3 { + return new Vec3(pos.x, pos.y, pos.z) +} + export const actionsList: Action[] = [ { name: 'chat', @@ -147,9 +155,31 @@ export const actionsList: Action[] = [ closeness: z.number().describe('How close to get to the player in blocks.').min(0), }), perform: mineflayer => async (player_name: string, closeness: number) => { + const getPlayerPos = () => { + const entity = mineflayer.bot.players[player_name]?.entity + return entity ? cloneVec3(entity.position) : null + } + + const selfStart = cloneVec3(mineflayer.bot.entity.position) + const targetStart = getPlayerPos() + const distanceToTargetBefore = targetStart ? selfStart.distanceTo(targetStart) : null + // TODO estimate time cost based on distance, trigger failure if time runs out - await skills.goToPlayer(mineflayer, player_name, closeness) - return `Arrived at player [${player_name}]` + const ok = await skills.goToPlayer(mineflayer, player_name, closeness) + + const selfEnd = cloneVec3(mineflayer.bot.entity.position) + const targetEnd = getPlayerPos() + const distanceToTargetAfter = targetEnd ? selfEnd.distanceTo(targetEnd) : null + + return { + ok, + target: { player_name, closeness }, + startPos: toCoord(selfStart), + endPos: toCoord(selfEnd), + movedDistance: selfStart.distanceTo(selfEnd), + distanceToTargetBefore, + distanceToTargetAfter, + } }, }, { @@ -197,8 +227,25 @@ export const actionsList: Action[] = [ closeness: z.number().describe('0 If want to be exactly at the position, otherwise a positive number in blocks for leniency.').min(0), }), perform: mineflayer => async (x: number, y: number, z: number, closeness: number) => { - await skills.goToPosition(mineflayer, x, y, z, closeness) - return `Arrived at coordinate [${x}, ${y}, ${z}]` + const selfStart = cloneVec3(mineflayer.bot.entity.position) + const targetVec = new Vec3(x, y, z) + const distanceToTargetBefore = selfStart.distanceTo(targetVec) + + const ok = await skills.goToPosition(mineflayer, x, y, z, closeness) + + const selfEnd = cloneVec3(mineflayer.bot.entity.position) + const distanceToTargetAfter = selfEnd.distanceTo(targetVec) + + return { + ok, + target: { x, y, z, closeness }, + startPos: toCoord(selfStart), + endPos: toCoord(selfEnd), + movedDistance: selfStart.distanceTo(selfEnd), + distanceToTargetBefore, + distanceToTargetAfter, + withinCloseness: distanceToTargetAfter <= closeness, + } }, }, { diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index 6faaeee5a..c10f868d2 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -36,6 +36,19 @@ interface QueuedEvent { reject: (err: Error) => void } +interface PlannerOutcomeSummary { + actionCount: number + okCount: number + errorCount: number + returnValue?: string + logs: string[] + updatedAt: number +} + +function truncateForPrompt(value: string, maxLength = 220): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}...` +} + export class Brain { private debugService: DebugService private readonly planner = new JavaScriptPlanner() @@ -49,6 +62,7 @@ export class Brain { private lastHumanChatAt = 0 private botUsername = '' private lastContextView: string | undefined + private lastPlannerOutcome: PlannerOutcomeSummary | undefined private conversationHistory: Message[] = [] constructor(private readonly deps: BrainDeps) { @@ -285,6 +299,15 @@ export class Brain { }, ) + this.lastPlannerOutcome = { + actionCount: runResult.actions.length, + okCount: runResult.actions.filter(item => item.ok).length, + errorCount: runResult.actions.filter(item => !item.ok).length, + returnValue: runResult.returnValue, + logs: runResult.logs.slice(-3), + updatedAt: Date.now(), + } + if (runResult.actions.length === 0 || runResult.actions.every(item => item.action.tool === 'skip')) { this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)') return @@ -347,7 +370,16 @@ export class Brain { parts.push(`[STATE] giveUp active (${remainingSec}s left). reason=${this.giveUpReason ?? 'unknown'}`) } - parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, mem, lastRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.') + if (this.lastPlannerOutcome) { + const ageMs = Date.now() - this.lastPlannerOutcome.updatedAt + const returnValue = truncateForPrompt(this.lastPlannerOutcome.returnValue ?? 'undefined') + const logs = this.lastPlannerOutcome.logs.length > 0 + ? this.lastPlannerOutcome.logs.map((line, index) => `#${index + 1} ${truncateForPrompt(line, 120)}`).join(' | ') + : '(none)' + parts.push(`[SCRIPT] Last eval ${ageMs}ms ago: return=${returnValue}; actions=${this.lastPlannerOutcome.actionCount} (ok=${this.lastPlannerOutcome.okCount}, err=${this.lastPlannerOutcome.errorCount}); logs=${logs}`) + } + + parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, mem, lastRun, prevRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.') return parts.join('\n\n') } diff --git a/services/minecraft/src/cognitive/conscious/js-planner.test.ts b/services/minecraft/src/cognitive/conscious/js-planner.test.ts index 40739b9a3..a8850ffc6 100644 --- a/services/minecraft/src/cognitive/conscious/js-planner.test.ts +++ b/services/minecraft/src/cognitive/conscious/js-planner.test.ts @@ -112,4 +112,38 @@ describe('JavaScriptPlanner', () => { await expect(planner.evaluate('while (true) {}', actions, globals, executeAction)).rejects.toThrow(/Script execution timed out/i) }) + + it('supports expectation guardrails on structured action telemetry', async () => { + const planner = new JavaScriptPlanner() + const executeAction = vi.fn(async () => ({ + ok: true, + movedDistance: 1.25, + distanceToTargetAfter: 1.5, + endPos: { x: 8, y: 64, z: 4 }, + })) + + const planned = await planner.evaluate(` + const nav = await goToPlayer({ player_name: "Alex", closeness: 2 }) + expect(nav.ok, "go failed") + expectMoved(1) + expectNear(2) + expectNear({ x: 7, y: 64, z: 4 }, 2) + `, actions, globals, executeAction) + + expect(planned.actions).toHaveLength(1) + expect(planned.actions[0]?.ok).toBe(true) + }) + + it('throws when expectation guardrail fails', async () => { + const planner = new JavaScriptPlanner() + const executeAction = vi.fn(async () => ({ + ok: true, + movedDistance: 0.1, + })) + + await expect(planner.evaluate(` + await goToPlayer({ player_name: "Alex", closeness: 2 }) + expectMoved(1, "did not move enough") + `, actions, globals, executeAction)).rejects.toThrow(/Expectation failed: did not move enough/i) + }) }) diff --git a/services/minecraft/src/cognitive/conscious/js-planner.ts b/services/minecraft/src/cognitive/conscious/js-planner.ts index d9af1d175..a62b47749 100644 --- a/services/minecraft/src/cognitive/conscious/js-planner.ts +++ b/services/minecraft/src/cognitive/conscious/js-planner.ts @@ -35,6 +35,13 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } +function isCoord(value: unknown): value is { x: number, y: number, z: number } { + return isRecord(value) + && typeof value.x === 'number' + && typeof value.y === 'number' + && typeof value.z === 'number' +} + function deepFreeze(value: T): T { if (!value || typeof value !== 'object') return value @@ -114,6 +121,10 @@ export class JavaScriptPlanner { ? undefined : inspect(result, { depth: 2, breakLength: 100 }) + if (isRecord(this.sandbox.lastRun)) { + this.sandbox.lastRun.returnValue = returnValue + } + return { actions: run.executed, logs: run.logs, @@ -143,9 +154,93 @@ export class JavaScriptPlanner { this.activeRun.logs.push(rendered) return rendered }) + this.defineGlobalTool('expect', (condition: unknown, message?: unknown) => { + if (condition) + return true + + const detail = typeof message === 'string' && message.trim().length > 0 + ? message + : 'Condition evaluated to false' + throw new Error(`Expectation failed: ${detail}`) + }) + this.defineGlobalTool('expectMoved', (minBlocks?: unknown, message?: unknown) => { + const threshold = typeof minBlocks === 'number' ? minBlocks : 0.5 + const telemetry = this.getLastActionResultRecord() + const movedDistance = typeof telemetry?.movedDistance === 'number' + ? telemetry.movedDistance + : null + + if (movedDistance === null) { + throw new Error('Expectation failed: expectMoved() requires last action result with movedDistance telemetry') + } + + if (movedDistance >= threshold) + return true + + const detail = typeof message === 'string' && message.trim().length > 0 + ? message + : `Expected movedDistance >= ${threshold}, got ${movedDistance}` + throw new Error(`Expectation failed: ${detail}`) + }) + this.defineGlobalTool('expectNear', (targetOrMaxDist?: unknown, maxDistOrMessage?: unknown, maybeMessage?: unknown) => { + const telemetry = this.getLastActionResultRecord() + + let target: { x: number, y: number, z: number } | null = null + let maxDist = 2 + let message: string | undefined + + if (isCoord(targetOrMaxDist)) { + target = { x: targetOrMaxDist.x, y: targetOrMaxDist.y, z: targetOrMaxDist.z } + if (typeof maxDistOrMessage === 'number') + maxDist = maxDistOrMessage + if (typeof maybeMessage === 'string') + message = maybeMessage + } + else { + if (typeof targetOrMaxDist === 'number') + maxDist = targetOrMaxDist + if (typeof maxDistOrMessage === 'string') + message = maxDistOrMessage + } + + let distance: number | null = null + if (target) { + const endPos = isCoord(telemetry?.endPos) ? telemetry.endPos : null + if (!endPos) { + throw new Error('Expectation failed: expectNear(target) requires last action result with endPos telemetry') + } + + const dx = endPos.x - target.x + const dy = endPos.y - target.y + const dz = endPos.z - target.z + distance = Math.sqrt(dx * dx + dy * dy + dz * dz) + } + else if (typeof telemetry?.distanceToTargetAfter === 'number') { + distance = telemetry.distanceToTargetAfter + } + + if (distance === null) { + throw new Error('Expectation failed: expectNear() requires target argument or last action distanceToTargetAfter telemetry') + } + + if (distance <= maxDist) + return true + + const detail = message ?? `Expected distance <= ${maxDist}, got ${distance}` + throw new Error(`Expectation failed: ${detail}`) + }) this.defineGlobalValue('mem', {}) } + private getLastActionResultRecord(): Record | null { + const lastAction = this.sandbox.lastAction + if (!isRecord(lastAction)) + return null + + const result = lastAction.result + return isRecord(result) ? result : null + } + private installActionTools(availableActions: Action[]): void { for (const action of availableActions) { this.defineGlobalTool(action.name, async (...args: unknown[]) => { diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts index adf637a57..cb0906fa5 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts @@ -108,6 +108,7 @@ You are an autonomous agent playing Minecraft. - Use \`await\` on tool calls when later logic depends on the result. - Globals refreshed every turn: \`snapshot\`, \`self\`, \`environment\`, \`social\`, \`threat\`, \`attention\`, \`autonomy\`, \`event\`, \`now\`. - Persistent globals: \`mem\` (cross-turn memory), \`lastRun\` (this run), \`prevRun\` (previous run), \`lastAction\` (latest action result), \`log(...)\`. + - Last script outcome is also echoed in the next turn as \`[SCRIPT]\` context (return value, action stats, and logs). - Maximum actions per turn: 5. # Environment & Global Semantics @@ -142,6 +143,7 @@ Call tool functions directly. Use \`await\` when branching on action outcomes. If you want to do nothing, call \`await skip()\`. You can also use \`use(toolName, paramsObject)\` for dynamic tool calls. +Use built-in guardrails to verify outcomes: \`expect(...)\`, \`expectMoved(...)\`, \`expectNear(...)\`. Examples: - \`await chat("hello")\` @@ -149,6 +151,29 @@ Examples: - \`const arrived = await goToPlayer({ player_name: "Alex", closeness: 2 }); if (!arrived) await chat("failed")\` - \`if (self.health < 10) await consume({ item_name: "bread" })\` - \`await skip()\` +- \`const nav = await goToCoordinate({ x: 12, y: 64, z: -5, closeness: 2 }); expect(nav.ok, "navigation failed"); expectMoved(0.8); expectNear(2.5)\` + +Guardrail semantics: +- \`expect(condition, message?)\`: throw if condition is falsy. +- \`expectMoved(minBlocks = 0.5, message?)\`: checks last action telemetry \`movedDistance\`. +- \`expectNear(targetOrMaxDist = 2, maxDist?, message?)\`: + - \`expectNear(2.5)\` uses last action telemetry \`distanceToTargetAfter\`. + - \`expectNear({ x, y, z }, 2)\` uses last action telemetry \`endPos\`. + +Common patterns: +- Follow + detach for exploration: + - \`await followPlayer({ player_name: "laggy_magpie", follow_dist: 2 })\` + - \`const nav = await goToCoordinate({ x: 120, y: 70, z: -30, closeness: 2 }) // detaches follow automatically\` + - \`expect(nav.ok, "failed to reach exploration point")\` +- Confirm movement before claiming progress: + - \`const r = await goToPlayer({ player_name: "Alex", closeness: 2 })\` + - \`expect(r.ok, "goToPlayer failed")\` + - \`expectMoved(1, "I did not actually move")\` + - \`expectNear(3, "still too far from player")\` +- Gaze as weak hint only: + - \`const gaze = environment.nearbyPlayersGaze.find(g => g.name === "Alex")\` + - \`if (event.type === "perception" && event.payload?.type === "chat_message" && gaze?.hitBlock)\` + - \` await goToCoordinate({ x: gaze.hitBlock.pos.x, y: gaze.hitBlock.pos.y, z: gaze.hitBlock.pos.z, closeness: 2 })\` # Usage Convention (Important) - Plan with \`mem.plan\`, execute in small steps, and verify each step before continuing. diff --git a/services/minecraft/src/cognitive/reflex/gaze.test.ts b/services/minecraft/src/cognitive/reflex/gaze.test.ts new file mode 100644 index 000000000..ecc846a08 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/gaze.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import { Vec3 } from 'vec3' + +import { rayTraceBlockFromEntity } from './gaze' + +describe('rayTraceBlockFromEntity', () => { + it('detects block when player looks downward', () => { + const bot = { + blockAt(pos: Vec3) { + if (pos.x === 0 && pos.y === 63 && pos.z === -3) { + return { + name: 'grass_block', + position: new Vec3(0, 63, -3), + } + } + return { name: 'air', position: pos } + }, + } as any + + const entity = { + type: 'player', + username: 'tester', + position: new Vec3(0, 64, 0), + yaw: 0, + pitch: Math.PI / 4, + } + + const result = rayTraceBlockFromEntity(bot, entity, { maxDistance: 8, step: 0.1 }) + expect(result.hitBlock?.name).toBe('grass_block') + expect(result.hitBlock?.pos).toEqual({ x: 0, y: 63, z: -3 }) + }) + + it('returns null hitBlock when no solid block is intersected', () => { + const bot = { + blockAt(pos: Vec3) { + return { name: 'air', position: pos } + }, + } as any + + const entity = { + type: 'player', + username: 'tester', + position: new Vec3(0, 64, 0), + yaw: 0, + pitch: -Math.PI / 4, + } + + const result = rayTraceBlockFromEntity(bot, entity, { maxDistance: 8, step: 0.1 }) + expect(result.hitBlock).toBeNull() + }) +}) diff --git a/services/minecraft/src/cognitive/reflex/gaze.ts b/services/minecraft/src/cognitive/reflex/gaze.ts index 48e77bb80..b6d12b096 100644 --- a/services/minecraft/src/cognitive/reflex/gaze.ts +++ b/services/minecraft/src/cognitive/reflex/gaze.ts @@ -24,7 +24,8 @@ export interface PlayerGazeResult { function directionFromYawPitch(yaw: number, pitch: number): Vec3Like { const x = -Math.sin(yaw) * Math.cos(pitch) - const y = Math.sin(pitch) + // In Minecraft, positive pitch means looking down (negative Y direction). + const y = -Math.sin(pitch) const z = -Math.cos(yaw) * Math.cos(pitch) return { x, y, z } }