nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(minecraft): capture and expose LLM input snapshot in REPL with source tracking

Browse Source 
Add LlmInputSnapshot interface tracking systemPrompt/userMessage/messages/conversationHistory/updatedAt/attempt, store lastLlmInputSnapshot in Brain and inject into RuntimeGlobals as llmInput, expose llmInput/llmMessages/llmSystemPrompt/llmUserMessage/llmConversationHistory as frozen readonly globals in JavaScriptPlanner sandbox, add source:'manual'|'llm' field to DebugReplResult to distinguish user-triggered vs autonomous
This commit is contained in:
Rin
2026-02-18 11:14:38 +08:00
committed by Neko Ayaka
parent cce492f685
commit f70f9fac46
7 changed files with 134 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/minecraft/src/cognitive/conscious/brain.ts
+77 -8
View File
@@ -48,6 +48,7 @@ interface PlannerOutcomeSummary {
}
interface DebugReplResult {
source: 'manual' | 'llm'
code: string
logs: string[]
actions: Array<{
@@ -63,6 +64,15 @@ interface DebugReplResult {
timestamp: number
}
interface LlmInputSnapshot {
systemPrompt: string
userMessage: string
messages: Message[]
conversationHistory: Message[]
updatedAt: number
attempt: number
}
function truncateForPrompt(value: string, maxLength = 220): string {
return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}...`
}
@@ -83,6 +93,7 @@ export class Brain {
private lastContextView: string | undefined
private lastPlannerOutcome: PlannerOutcomeSummary | undefined
private conversationHistory: Message[] = []
private lastLlmInputSnapshot: LlmInputSnapshot | null = null
constructor(private readonly deps: BrainDeps) {
this.debugService = DebugService.getInstance()
@@ -154,6 +165,7 @@ export class Brain {
timestamp: Date.now(),
},
snapshot: snapshot as unknown as Record<string, unknown>,
llmInput: this.lastLlmInputSnapshot,
},
)
@@ -167,6 +179,7 @@ export class Brain {
const startedAt = Date.now()
if (this.isProcessing || this.isReplEvaluating) {
return {
source: 'manual',
code,
logs: [],
actions: [],
@@ -196,6 +209,7 @@ export class Brain {
timestamp: Date.now(),
},
snapshot: snapshot as unknown as Record<string, unknown>,
llmInput: this.lastLlmInputSnapshot,
},
async (action: ActionInstruction) => {
const actionDef = actionDefs.get(action.tool)
@@ -206,15 +220,10 @@ export class Brain {
)
return {
source: 'manual',
code,
logs: runResult.logs,
actions: runResult.actions.map(item => ({
tool: item.action.tool,
params: item.action.params,
ok: item.ok,
result: item.result === undefined ? undefined : (typeof item.result === 'string' ? item.result : JSON.stringify(item.result)),
error: item.error,
})),
actions: this.toDebugReplActions(runResult.actions),
returnValue: runResult.returnValue,
durationMs: Date.now() - startedAt,
timestamp: Date.now(),
@@ -222,6 +231,7 @@ export class Brain {
}
catch (err) {
return {
source: 'manual',
code,
logs: [],
actions: [],
@@ -244,6 +254,25 @@ export class Brain {
)
}
private toDebugReplActions(actions: Array<{
action: ActionInstruction
ok: boolean
result?: unknown
error?: string
}>): DebugReplResult['actions'] {
return actions.map(item => ({
tool: item.action.tool,
params: item.action.params,
ok: item.ok,
result: item.result === undefined ? undefined : (typeof item.result === 'string' ? item.result : JSON.stringify(item.result)),
error: item.error,
}))
}
private cloneMessages(messages: Message[]): Message[] {
return JSON.parse(JSON.stringify(messages)) as Message[]
}
// --- Event Queue Logic ---
private async enqueueEvent(bot: MineflayerWithAgents, event: BotEvent): Promise<void> {
@@ -325,6 +354,14 @@ export class Brain {
...this.conversationHistory,
{ role: 'user', content: userMessage },
]
this.lastLlmInputSnapshot = {
systemPrompt,
userMessage,
messages: this.cloneMessages(messages),
conversationHistory: this.cloneMessages(this.conversationHistory),
updatedAt: Date.now(),
attempt,
}
const traceStart = Date.now()
@@ -400,7 +437,11 @@ export class Brain {
const runResult = await this.planner.evaluate(
result,
this.deps.taskExecutor.getAvailableActions(),
{ event, snapshot: snapshot as unknown as Record<string, unknown> },
{
event,
snapshot: snapshot as unknown as Record<string, unknown>,
llmInput: this.lastLlmInputSnapshot,
},
async (action: ActionInstruction) => {
if (action.tool === 'chat' && !this.shouldAllowChatForEvent(event, snapshot.self.health)) {
return 'Chat suppressed: no direct user prompt for chat this turn'
@@ -435,10 +476,29 @@ export class Brain {
}
if (runResult.actions.length === 0 || runResult.actions.every(item => item.action.tool === 'skip')) {
this.debugService.emit('debug:repl_result', {
source: 'llm',
code: result,
logs: runResult.logs,
actions: this.toDebugReplActions(runResult.actions),
returnValue: runResult.returnValue,
durationMs: 0,
timestamp: Date.now(),
})
this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)')
return
}
this.debugService.emit('debug:repl_result', {
source: 'llm',
code: result,
logs: runResult.logs,
actions: this.toDebugReplActions(runResult.actions),
returnValue: runResult.returnValue,
durationMs: 0,
timestamp: Date.now(),
})
this.deps.logger.log('INFO', `Brain: Executed ${runResult.actions.length} action(s)`, {
actions: runResult.actions.map(item => ({
tool: item.action.tool,
@@ -452,6 +512,15 @@ export class Brain {
}
catch (err) {
this.deps.logger.withError(err).error('Brain: Failed to execute decision')
this.debugService.emit('debug:repl_result', {
source: 'llm',
code: result,
logs: [],
actions: [],
error: toErrorMessage(err),
durationMs: 0,
timestamp: Date.now(),
})
void this.enqueueEvent(bot, {
type: 'feedback',
payload: { status: 'failure', error: toErrorMessage(err) },
services/minecraft/src/cognitive/conscious/js-planner.test.ts
+17
View File
@@ -38,6 +38,14 @@ describe('javaScriptPlanner', () => {
threat: {},
attention: {},
},
llmInput: {
systemPrompt: 'system prompt',
userMessage: 'latest user message',
messages: [{ role: 'user', content: 'hello' }],
conversationHistory: [{ role: 'assistant', content: 'previous reply' }],
updatedAt: Date.now(),
attempt: 1,
},
} as any
it('maps positional/object args and executes tools in order', async () => {
@@ -155,11 +163,20 @@ describe('javaScriptPlanner', () => {
expect(names).toContain('mem')
expect(names).toContain('chat')
expect(names).toContain('goToPlayer')
expect(names).toContain('llmInput')
expect(names).toContain('llmUserMessage')
const mem = descriptors.find(d => d.name === 'mem')
expect(mem?.readonly).toBe(false)
})
it('exposes llm input globals to scripts', async () => {
const planner = new JavaScriptPlanner()
const executeAction = vi.fn(async action => `ok:${action.tool}`)
const planned = await planner.evaluate('await chat("llm=" + llmUserMessage)', actions, globals, executeAction)
expect(planned.actions[0]?.action).toEqual({ tool: 'chat', params: { message: 'llm=latest user message' } })
})
it('detects expression-friendly REPL inputs', () => {
const planner = new JavaScriptPlanner()
expect(planner.canEvaluateAsExpression('2 + 3')).toBe(true)
services/minecraft/src/cognitive/conscious/js-planner.ts
+25 -1
View File
@@ -62,6 +62,14 @@ function toStructuredClone<T>(value: T): T {
export interface RuntimeGlobals {
event: BotEvent
snapshot: Record<string, unknown>
llmInput?: {
systemPrompt: string
userMessage: string
messages: unknown[]
conversationHistory: unknown[]
updatedAt: number
attempt: number
} | null
}
export interface JavaScriptRunResult {
@@ -177,6 +185,11 @@ export class JavaScriptPlanner {
{ name: 'threat', kind: 'object', readonly: true },
{ name: 'attention', kind: 'object', readonly: true },
{ name: 'autonomy', kind: 'object', readonly: true },
{ name: 'llmInput', kind: 'object', readonly: true },
{ name: 'llmMessages', kind: 'object', readonly: true },
{ name: 'llmSystemPrompt', kind: 'string', readonly: true },
{ name: 'llmUserMessage', kind: 'string', readonly: true },
{ name: 'llmConversationHistory', kind: 'object', readonly: true },
{ name: 'mem', kind: 'object', readonly: false },
{ name: 'lastRun', kind: 'object', readonly: true },
{ name: 'prevRun', kind: 'object', readonly: true },
@@ -193,6 +206,11 @@ export class JavaScriptPlanner {
threat: (globals.snapshot as Record<string, unknown>)?.threat,
attention: (globals.snapshot as Record<string, unknown>)?.attention,
autonomy: (globals.snapshot as Record<string, unknown>)?.autonomy,
llmInput: globals.llmInput ?? null,
llmMessages: globals.llmInput?.messages ?? [],
llmSystemPrompt: globals.llmInput?.systemPrompt ?? '',
llmUserMessage: globals.llmInput?.userMessage ?? '',
llmConversationHistory: globals.llmInput?.conversationHistory ?? [],
mem: this.sandbox.mem,
lastRun: this.sandbox.lastRun,
prevRun: this.sandbox.prevRun,
@@ -342,6 +360,7 @@ export class JavaScriptPlanner {
private bindRuntimeGlobals(globals: RuntimeGlobals, run: ActivePlannerRun): void {
const snapshot = deepFreeze(toStructuredClone(globals.snapshot))
const event = deepFreeze(toStructuredClone(globals.event))
const llmInput = deepFreeze(toStructuredClone(globals.llmInput ?? null))
this.sandbox.prevRun = this.sandbox.lastRun ?? null
this.sandbox.snapshot = snapshot
@@ -353,6 +372,11 @@ export class JavaScriptPlanner {
this.sandbox.threat = snapshot.threat
this.sandbox.attention = snapshot.attention
this.sandbox.autonomy = snapshot.autonomy
this.sandbox.llmInput = llmInput
this.sandbox.llmMessages = llmInput?.messages ?? []
this.sandbox.llmSystemPrompt = llmInput?.systemPrompt ?? ''
this.sandbox.llmUserMessage = llmInput?.userMessage ?? ''
this.sandbox.llmConversationHistory = llmInput?.conversationHistory ?? []
this.sandbox.lastRun = {
actions: run.executed,
logs: run.logs,
@@ -463,7 +487,7 @@ export class JavaScriptPlanner {
const parsed = action.schema.safeParse(params)
if (!parsed.success) {
const details = parsed.error.issues
.map((issue: { path: Array<string | number>, message: string }) => `${issue.path.join('.') || 'root'}: ${issue.message}`)
.map(issue => `${issue.path.map(item => String(item)).join('.') || 'root'}: ${issue.message}`)
.join('; ')
return {
error: `Invalid tool parameters for ${tool}: ${details}`,
services/minecraft/src/cognitive/index.ts
+1
View File
@@ -36,6 +36,7 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug
const code = command.payload?.code
if (typeof code !== 'string') {
debugService.emit('debug:repl_result', {
source: 'manual',
code: '',
logs: [],
actions: [],
services/minecraft/src/debug/types.ts
+1
View File
@@ -152,6 +152,7 @@ export interface ReplStateEvent {
}
export interface ReplExecutionResultEvent {
source: 'manual' | 'llm'
code: string
logs: string[]
actions: Array<{
services/minecraft/src/debug/web/app.js
+5 -2
View File
@@ -1048,6 +1048,7 @@ class ReplPanel {
const code = this.elements.codeInput?.value ?? ''
if (!code.trim()) {
this.results.unshift({
source: 'manual',
code: '',
logs: [],
actions: [],
@@ -1136,6 +1137,8 @@ class ReplPanel {
this.elements.resultList.innerHTML = this.results.map((result) => {
const isError = !!result.error
const source = result.source === 'llm' ? 'llm' : 'manual'
const sourceLabel = source === 'llm' ? 'LLM' : 'MANUAL'
const actionSummary = Array.isArray(result.actions) && result.actions.length > 0
? result.actions.map((action) => {
const status = action.ok ? 'ok' : 'error'
@@ -1149,9 +1152,9 @@ class ReplPanel {
const time = new Date(result.timestamp || Date.now()).toLocaleTimeString()
return `
<div class="repl-result-card ${isError ? 'error' : ''}">
<div class="repl-result-card source-${source} ${isError ? 'error' : ''}">
<div class="repl-result-meta">
<span>${time}</span>
<span>${time} · ${sourceLabel}</span>
<span>${Number.isFinite(result.durationMs) ? `${result.durationMs}ms` : '-'}</span>
</div>
<div class="repl-result-section">
services/minecraft/src/debug/web/styles.css
+8
View File
@@ -310,6 +310,14 @@ button:hover,
font-size: 11px;
}
.repl-result-card.source-manual {
border-color: #2f81f7;
}
.repl-result-card.source-llm {
border-color: #d29922;
}
.repl-result-card.error {
border-color: var(--accent-error);
}