diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index 583e15a40..ead99c573 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -48,6 +48,7 @@ interface PlannerOutcomeSummary { } interface DebugReplResult { + source: 'manual' | 'llm' code: string logs: string[] actions: Array<{ @@ -63,6 +64,15 @@ interface DebugReplResult { timestamp: number } +interface LlmInputSnapshot { + systemPrompt: string + userMessage: string + messages: Message[] + conversationHistory: Message[] + updatedAt: number + attempt: number +} + function truncateForPrompt(value: string, maxLength = 220): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}...` } @@ -83,6 +93,7 @@ export class Brain { private lastContextView: string | undefined private lastPlannerOutcome: PlannerOutcomeSummary | undefined private conversationHistory: Message[] = [] + private lastLlmInputSnapshot: LlmInputSnapshot | null = null constructor(private readonly deps: BrainDeps) { this.debugService = DebugService.getInstance() @@ -154,6 +165,7 @@ export class Brain { timestamp: Date.now(), }, snapshot: snapshot as unknown as Record, + llmInput: this.lastLlmInputSnapshot, }, ) @@ -167,6 +179,7 @@ export class Brain { const startedAt = Date.now() if (this.isProcessing || this.isReplEvaluating) { return { + source: 'manual', code, logs: [], actions: [], @@ -196,6 +209,7 @@ export class Brain { timestamp: Date.now(), }, snapshot: snapshot as unknown as Record, + llmInput: this.lastLlmInputSnapshot, }, async (action: ActionInstruction) => { const actionDef = actionDefs.get(action.tool) @@ -206,15 +220,10 @@ export class Brain { ) return { + source: 'manual', code, logs: runResult.logs, - actions: runResult.actions.map(item => ({ - tool: item.action.tool, - params: item.action.params, - ok: item.ok, - result: item.result === undefined ? undefined : (typeof item.result === 'string' ? item.result : JSON.stringify(item.result)), - error: item.error, - })), + actions: this.toDebugReplActions(runResult.actions), returnValue: runResult.returnValue, durationMs: Date.now() - startedAt, timestamp: Date.now(), @@ -222,6 +231,7 @@ export class Brain { } catch (err) { return { + source: 'manual', code, logs: [], actions: [], @@ -244,6 +254,25 @@ export class Brain { ) } + private toDebugReplActions(actions: Array<{ + action: ActionInstruction + ok: boolean + result?: unknown + error?: string + }>): DebugReplResult['actions'] { + return actions.map(item => ({ + tool: item.action.tool, + params: item.action.params, + ok: item.ok, + result: item.result === undefined ? undefined : (typeof item.result === 'string' ? item.result : JSON.stringify(item.result)), + error: item.error, + })) + } + + private cloneMessages(messages: Message[]): Message[] { + return JSON.parse(JSON.stringify(messages)) as Message[] + } + // --- Event Queue Logic --- private async enqueueEvent(bot: MineflayerWithAgents, event: BotEvent): Promise { @@ -325,6 +354,14 @@ export class Brain { ...this.conversationHistory, { role: 'user', content: userMessage }, ] + this.lastLlmInputSnapshot = { + systemPrompt, + userMessage, + messages: this.cloneMessages(messages), + conversationHistory: this.cloneMessages(this.conversationHistory), + updatedAt: Date.now(), + attempt, + } const traceStart = Date.now() @@ -400,7 +437,11 @@ export class Brain { const runResult = await this.planner.evaluate( result, this.deps.taskExecutor.getAvailableActions(), - { event, snapshot: snapshot as unknown as Record }, + { + event, + snapshot: snapshot as unknown as Record, + llmInput: this.lastLlmInputSnapshot, + }, async (action: ActionInstruction) => { if (action.tool === 'chat' && !this.shouldAllowChatForEvent(event, snapshot.self.health)) { return 'Chat suppressed: no direct user prompt for chat this turn' @@ -435,10 +476,29 @@ export class Brain { } if (runResult.actions.length === 0 || runResult.actions.every(item => item.action.tool === 'skip')) { + this.debugService.emit('debug:repl_result', { + source: 'llm', + code: result, + logs: runResult.logs, + actions: this.toDebugReplActions(runResult.actions), + returnValue: runResult.returnValue, + durationMs: 0, + timestamp: Date.now(), + }) this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)') return } + this.debugService.emit('debug:repl_result', { + source: 'llm', + code: result, + logs: runResult.logs, + actions: this.toDebugReplActions(runResult.actions), + returnValue: runResult.returnValue, + durationMs: 0, + timestamp: Date.now(), + }) + this.deps.logger.log('INFO', `Brain: Executed ${runResult.actions.length} action(s)`, { actions: runResult.actions.map(item => ({ tool: item.action.tool, @@ -452,6 +512,15 @@ export class Brain { } catch (err) { this.deps.logger.withError(err).error('Brain: Failed to execute decision') + this.debugService.emit('debug:repl_result', { + source: 'llm', + code: result, + logs: [], + actions: [], + error: toErrorMessage(err), + durationMs: 0, + timestamp: Date.now(), + }) void this.enqueueEvent(bot, { type: 'feedback', payload: { status: 'failure', error: toErrorMessage(err) }, diff --git a/services/minecraft/src/cognitive/conscious/js-planner.test.ts b/services/minecraft/src/cognitive/conscious/js-planner.test.ts index edeaa62aa..6c23b16d2 100644 --- a/services/minecraft/src/cognitive/conscious/js-planner.test.ts +++ b/services/minecraft/src/cognitive/conscious/js-planner.test.ts @@ -38,6 +38,14 @@ describe('javaScriptPlanner', () => { threat: {}, attention: {}, }, + llmInput: { + systemPrompt: 'system prompt', + userMessage: 'latest user message', + messages: [{ role: 'user', content: 'hello' }], + conversationHistory: [{ role: 'assistant', content: 'previous reply' }], + updatedAt: Date.now(), + attempt: 1, + }, } as any it('maps positional/object args and executes tools in order', async () => { @@ -155,11 +163,20 @@ describe('javaScriptPlanner', () => { expect(names).toContain('mem') expect(names).toContain('chat') expect(names).toContain('goToPlayer') + expect(names).toContain('llmInput') + expect(names).toContain('llmUserMessage') const mem = descriptors.find(d => d.name === 'mem') expect(mem?.readonly).toBe(false) }) + it('exposes llm input globals to scripts', async () => { + const planner = new JavaScriptPlanner() + const executeAction = vi.fn(async action => `ok:${action.tool}`) + const planned = await planner.evaluate('await chat("llm=" + llmUserMessage)', actions, globals, executeAction) + expect(planned.actions[0]?.action).toEqual({ tool: 'chat', params: { message: 'llm=latest user message' } }) + }) + it('detects expression-friendly REPL inputs', () => { const planner = new JavaScriptPlanner() expect(planner.canEvaluateAsExpression('2 + 3')).toBe(true) diff --git a/services/minecraft/src/cognitive/conscious/js-planner.ts b/services/minecraft/src/cognitive/conscious/js-planner.ts index 558a4975a..a39e7fd96 100644 --- a/services/minecraft/src/cognitive/conscious/js-planner.ts +++ b/services/minecraft/src/cognitive/conscious/js-planner.ts @@ -62,6 +62,14 @@ function toStructuredClone(value: T): T { export interface RuntimeGlobals { event: BotEvent snapshot: Record + llmInput?: { + systemPrompt: string + userMessage: string + messages: unknown[] + conversationHistory: unknown[] + updatedAt: number + attempt: number + } | null } export interface JavaScriptRunResult { @@ -177,6 +185,11 @@ export class JavaScriptPlanner { { name: 'threat', kind: 'object', readonly: true }, { name: 'attention', kind: 'object', readonly: true }, { name: 'autonomy', kind: 'object', readonly: true }, + { name: 'llmInput', kind: 'object', readonly: true }, + { name: 'llmMessages', kind: 'object', readonly: true }, + { name: 'llmSystemPrompt', kind: 'string', readonly: true }, + { name: 'llmUserMessage', kind: 'string', readonly: true }, + { name: 'llmConversationHistory', kind: 'object', readonly: true }, { name: 'mem', kind: 'object', readonly: false }, { name: 'lastRun', kind: 'object', readonly: true }, { name: 'prevRun', kind: 'object', readonly: true }, @@ -193,6 +206,11 @@ export class JavaScriptPlanner { threat: (globals.snapshot as Record)?.threat, attention: (globals.snapshot as Record)?.attention, autonomy: (globals.snapshot as Record)?.autonomy, + llmInput: globals.llmInput ?? null, + llmMessages: globals.llmInput?.messages ?? [], + llmSystemPrompt: globals.llmInput?.systemPrompt ?? '', + llmUserMessage: globals.llmInput?.userMessage ?? '', + llmConversationHistory: globals.llmInput?.conversationHistory ?? [], mem: this.sandbox.mem, lastRun: this.sandbox.lastRun, prevRun: this.sandbox.prevRun, @@ -342,6 +360,7 @@ export class JavaScriptPlanner { private bindRuntimeGlobals(globals: RuntimeGlobals, run: ActivePlannerRun): void { const snapshot = deepFreeze(toStructuredClone(globals.snapshot)) const event = deepFreeze(toStructuredClone(globals.event)) + const llmInput = deepFreeze(toStructuredClone(globals.llmInput ?? null)) this.sandbox.prevRun = this.sandbox.lastRun ?? null this.sandbox.snapshot = snapshot @@ -353,6 +372,11 @@ export class JavaScriptPlanner { this.sandbox.threat = snapshot.threat this.sandbox.attention = snapshot.attention this.sandbox.autonomy = snapshot.autonomy + this.sandbox.llmInput = llmInput + this.sandbox.llmMessages = llmInput?.messages ?? [] + this.sandbox.llmSystemPrompt = llmInput?.systemPrompt ?? '' + this.sandbox.llmUserMessage = llmInput?.userMessage ?? '' + this.sandbox.llmConversationHistory = llmInput?.conversationHistory ?? [] this.sandbox.lastRun = { actions: run.executed, logs: run.logs, @@ -463,7 +487,7 @@ export class JavaScriptPlanner { const parsed = action.schema.safeParse(params) if (!parsed.success) { const details = parsed.error.issues - .map((issue: { path: Array, message: string }) => `${issue.path.join('.') || 'root'}: ${issue.message}`) + .map(issue => `${issue.path.map(item => String(item)).join('.') || 'root'}: ${issue.message}`) .join('; ') return { error: `Invalid tool parameters for ${tool}: ${details}`, diff --git a/services/minecraft/src/cognitive/index.ts b/services/minecraft/src/cognitive/index.ts index 26ea45e78..4770f244b 100644 --- a/services/minecraft/src/cognitive/index.ts +++ b/services/minecraft/src/cognitive/index.ts @@ -36,6 +36,7 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug const code = command.payload?.code if (typeof code !== 'string') { debugService.emit('debug:repl_result', { + source: 'manual', code: '', logs: [], actions: [], diff --git a/services/minecraft/src/debug/types.ts b/services/minecraft/src/debug/types.ts index 7d5b676eb..fec24c6e2 100644 --- a/services/minecraft/src/debug/types.ts +++ b/services/minecraft/src/debug/types.ts @@ -152,6 +152,7 @@ export interface ReplStateEvent { } export interface ReplExecutionResultEvent { + source: 'manual' | 'llm' code: string logs: string[] actions: Array<{ diff --git a/services/minecraft/src/debug/web/app.js b/services/minecraft/src/debug/web/app.js index a4c9cfc4e..ae014bfdf 100644 --- a/services/minecraft/src/debug/web/app.js +++ b/services/minecraft/src/debug/web/app.js @@ -1048,6 +1048,7 @@ class ReplPanel { const code = this.elements.codeInput?.value ?? '' if (!code.trim()) { this.results.unshift({ + source: 'manual', code: '', logs: [], actions: [], @@ -1136,6 +1137,8 @@ class ReplPanel { this.elements.resultList.innerHTML = this.results.map((result) => { const isError = !!result.error + const source = result.source === 'llm' ? 'llm' : 'manual' + const sourceLabel = source === 'llm' ? 'LLM' : 'MANUAL' const actionSummary = Array.isArray(result.actions) && result.actions.length > 0 ? result.actions.map((action) => { const status = action.ok ? 'ok' : 'error' @@ -1149,9 +1152,9 @@ class ReplPanel { const time = new Date(result.timestamp || Date.now()).toLocaleTimeString() return ` -
+
- ${time} + ${time} ยท ${sourceLabel} ${Number.isFinite(result.durationMs) ? `${result.durationMs}ms` : '-'}
diff --git a/services/minecraft/src/debug/web/styles.css b/services/minecraft/src/debug/web/styles.css index fef3f3b24..07c2cd333 100644 --- a/services/minecraft/src/debug/web/styles.css +++ b/services/minecraft/src/debug/web/styles.css @@ -310,6 +310,14 @@ button:hover, font-size: 11px; } +.repl-result-card.source-manual { + border-color: #2f81f7; +} + +.repl-result-card.source-llm { + border-color: #d29922; +} + .repl-result-card.error { border-color: var(--accent-error); }