diff --git a/services/minecraft/docs/repl-self-evolution-roadmap.md b/services/minecraft/docs/repl-self-evolution-roadmap.md new file mode 100644 index 000000000..28f0788b6 --- /dev/null +++ b/services/minecraft/docs/repl-self-evolution-roadmap.md @@ -0,0 +1,252 @@ +# REPL Self-Evolution Roadmap (Discussion Draft) + +## Goal + +Turn the current JS planner REPL into a "poor man's coding agent" environment that can: + +1. Inspect its own runtime and project code. +2. Read docs (especially Mineflayer docs) from within the REPL. +3. Propose and eventually apply tool/runtime improvements safely. +4. Register event hooks and optional background daemons for long-lived behaviors. + +No implementation details in this draft are final; this is a planning artifact for review. + +--- + +## Design Principles + +- **Safety-first by default**: proposal mode before apply mode. +- **Auditability**: every introspection and mutation action is logged and replayable. +- **Layered capabilities**: observation -> proposal -> gated execution. +- **Deterministic core loop**: keep turn-based planner behavior stable. +- **Minimal trusted surface**: expose small, explicit APIs rather than raw Node/system access. + +--- + +## Capability Areas + +## 1) Self-Introspection Interface (REPL APIs) + +Expose a read-oriented introspection namespace in REPL globals. + +Proposed APIs: + +- `introspect.runtime()` -> model, tool list, limits, feature flags, queue status. +- `introspect.tools()` -> current action/tool signatures and schemas. +- `introspect.memory()` -> mem summary + size stats. +- `introspect.last()` -> previous script/action results. +- `introspect.health()` -> basic diagnostics (event lag, retries, failures). + +Notes: + +- Start read-only. +- Keep outputs compact and structured. + +## 2) Project Code Reading Interface + +Expose controlled repository browsing from REPL without full shell. + +Proposed APIs: + +- `repo.list(path = ".", opts?)` -> file/dir listing (allowlisted roots). +- `repo.read(path, opts?)` -> file snippet with line ranges and size cap. +- `repo.search(query, opts?)` -> ripgrep-backed search with result caps. +- `repo.symbol(path, name)` -> optional simple symbol lookup (later phase). + +Guardrails: + +- Read-only initially. +- Max file size / line range limits. +- Block secrets and ignored paths (`.env`, keys, tokens, node_modules, build outputs). + +## 3) Mineflayer Knowledge Surface + +Provide docs/refs to help the agent compose better behavior/tool proposals. + +Options: + +- Local curated docs bundle (recommended for deterministic behavior). +- Indexed snippets for common APIs (movement, inventory, entities, events). +- Optional "doc cards" in prompt context for high-frequency use. + +REPL API: + +- `docs.find("pathfinder")` +- `docs.read("mineflayer.bot.chat")` +- `docs.example("collectBlock")` + +## 4) Self-Evolution Workflow (Proposal First) + +Add a governance pipeline before any self-modification: + +1. Agent inspects code/docs/runtime. +2. Agent emits a **Change Proposal** object: + - rationale + - files impacted + - risks + - test plan + - rollback plan +3. Human reviews proposal. +4. Optional apply step (future): patch generation + tests + review gate. + +Proposed REPL APIs: + +- `evolve.propose(spec)` +- `evolve.listProposals()` +- `evolve.getProposal(id)` +- `evolve.reject(id, reason)` / `evolve.approve(id)` (human-mediated) + +## 5) Event Hooks + +Support persistent declarative hooks that trigger scripts on conditions. + +Hook model: + +- Trigger: chat/perception/feedback/time tick/custom. +- Condition: JS predicate over `ctx`/`last`/`mem`. +- Action: script body or tool calls. +- Policy: debounce, cooldown, max executions, priority. + +API sketch: + +- `hooks.register({ name, on, when, script, policy })` +- `hooks.list()` +- `hooks.disable(name)` / `hooks.enable(name)` / `hooks.remove(name)` + +Guardrails: + +- Prevent recursive storms (hook triggers caused by own outputs). +- Global execution budget per minute. +- Trace each hook invocation. + +## 6) Background Daemons + +Allow optional long-running tasks for monitoring/planning loops. + +Potential uses: + +- Patrol/scan loops. +- Inventory housekeeping. +- Threat watcher with alerts. +- Social etiquette responder. + +API sketch: + +- `daemon.start({ name, script, intervalMs, policy })` +- `daemon.stop(name)` +- `daemon.status(name)` / `daemon.list()` + +Constraints: + +- Strict per-daemon CPU/time budgets. +- Kill switch and watchdog timeout. +- No direct world mutation unless via validated tool intents. + +--- + +## Architecture Sketch + +## REPL Context Modules + +- `ctx` (readonly, per-turn injected) +- `last` (readonly, last outcome) +- `mem` (persistent writable) +- `introspect` (readonly API) +- `repo` (read-only at first) +- `docs` (read-only retrieval) +- `hooks` (managed registration) +- `daemon` (managed lifecycle) +- `evolve` (proposal orchestration) + +## Runtime Services (outside REPL sandbox) + +- Introspection service +- Repo/document index service +- Hook scheduler +- Daemon supervisor +- Proposal registry + audit log + +--- + +## Rollout Plan + +## Phase 0 - Foundation Hardening + +- Finalize `ctx/last/mem` conventions and docs. +- Add telemetry on script size, execution time, action count. +- Add rate limits and anti-loop protections. + +## Phase 1 - Read-Only Introspection + Repo/Docs Read + +- Implement `introspect.*`, `repo.list/read/search`, `docs.find/read`. +- Add allowlist + secret/path guards. +- Add prompt conventions for using these APIs responsibly. + +## Phase 2 - Proposal-Only Self-Evolution + +- Implement `evolve.propose` and proposal registry. +- Define proposal schema + scoring rubric. +- Add review UI/log stream hooks. + +## Phase 3 - Hooks + Daemons (Constrained) + +- Implement managed hooks with cooldown/budgets. +- Implement supervised daemon runner. +- Add kill switch + watchdog + per-feature flags. + +## Phase 4 - Assisted Apply (Optional) + +- Agent can prepare patch candidates + test commands. +- Human approval required before applying. +- Automatic rollback metadata and validation report. + +--- + +## Prompt/Policy Conventions to Add + +- Prefer world tools for immediate actions. +- Use `repo/docs/introspect` for understanding before proposing changes. +- Never assume write privileges; use `evolve.propose`. +- Keep hook/daemon logic idempotent and bounded. +- Explain intent in `mem` notes before major behavior changes. + +--- + +## Risk Register + +- **Runaway autonomy**: hooks/daemons creating infinite loops. +- **Context bloat**: huge introspection outputs inflating token use. +- **Unsafe self-editing**: bad proposals that degrade behavior. +- **Secret leakage**: repo read accidentally exposing credentials. +- **Operational complexity**: too many async subsystems reducing predictability. + +Mitigations: + +- Hard limits + quotas + cooldowns. +- Strong allowlists and redaction layers. +- Proposal-only first, human gates later. +- Feature flags for every new subsystem. +- Rich tracing and one-command emergency disable. + +--- + +## Open Questions (For Discussion) + +1. Should proposal approval live in chat only, or also in a local UI/CLI queue? +2. How much repo scope should be readable by default? +3. Should hooks/daemons be persisted across restarts? +4. Do we want separate "safe mode" and "experimental mode" presets? +5. What exact success metrics define "self-evolution is helping"? + +--- + +## Suggested First Milestone + +Implement only: + +- `introspect.runtime/tools/health` +- `repo.list/read/search` (read-only, strict limits) +- `docs.find/read` (curated Mineflayer docs) +- `evolve.propose` registry (no apply) + +This gives strong value quickly while keeping risk low. diff --git a/services/minecraft/src/cognitive/index.ts b/services/minecraft/src/cognitive/index.ts index 385e51d75..b9f82e659 100644 --- a/services/minecraft/src/cognitive/index.ts +++ b/services/minecraft/src/cognitive/index.ts @@ -2,7 +2,6 @@ import type { MineflayerPlugin } from '../libs/mineflayer' import type { CognitiveEngineOptions, MineflayerWithAgents } from './types' import { config } from '../composables/config' -import { DebugService } from '../debug' import { ChatMessageHandler } from '../libs/mineflayer' import { createAgentContainer } from './container' import { computeNearbyPlayerGaze } from './reflex/gaze' @@ -70,21 +69,13 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug }) }) - // Resolve EventBus and subscribe to forward events to debug timeline + // Resolve EventBus for message handling const eventBus = container.resolve('eventBus') - eventBus.subscribe('*', (event) => { - // Forward to debug service for timeline visualization - DebugService.getInstance().emitTrace({ - id: event.id, - traceId: event.traceId, - parentId: event.parentId, - type: event.type, - payload: event.payload, - timestamp: event.timestamp, - source: event.source, - }) - }) + // NOTICE: EventBus trace forwarding disabled - trace logs removed to reduce noise + // All events from EventBus were being forwarded to DebugService as trace events, + // causing thousands of 'raw:sighted:entity_moved' entries in the logs. + // Conscious layer (LLM) events are still logged separately. // Set message handling via EventBus const chatHandler = new ChatMessageHandler(bot.username) diff --git a/services/minecraft/src/debug/server.ts b/services/minecraft/src/debug/server.ts index e51e28bfe..0deddcf2c 100644 --- a/services/minecraft/src/debug/server.ts +++ b/services/minecraft/src/debug/server.ts @@ -183,6 +183,11 @@ export class DebugServer { return } + if (req.method === 'GET' && req.url?.startsWith('/api/logs')) { + this.handleLogsApi(req, res) + return + } + if (req.method === 'GET' && (req.url === '/viewer' || req.url?.startsWith('/viewer?'))) { const html = createViewerHtml('http://localhost:3007') res.writeHead(200, { @@ -338,13 +343,19 @@ export class DebugServer { } private persistEvent(event: ServerEvent): void { - if (this.logStream) { - try { - this.logStream.write(`${JSON.stringify(event)}\n`) - } - catch (err) { - console.error('Failed to write to log file', err) - } + const persistableTypes: ServerEvent['type'][] = ['log', 'llm', 'blackboard', 'queue', 'trace', 'trace_batch', 'reflex'] + + if (!persistableTypes.includes(event.type)) + return + + if (!this.logStream) + return + + try { + this.logStream.write(`${JSON.stringify(event)}\n`) + } + catch (err) { + console.error('Failed to write to log file', err) } } @@ -359,4 +370,55 @@ export class DebugServer { private generateClientId(): string { return `client-${Date.now()}-${Math.random().toString(36).substring(2, 9)}` } + + private handleLogsApi(req: IncomingMessage, res: http.ServerResponse): void { + const url = new URL(req.url || '/api/logs', 'http://localhost') + const logsDir = path.join(process.cwd(), 'logs') + + // GET /api/logs -> list files + if (!url.searchParams.has('file')) { + const files = fs.existsSync(logsDir) + ? fs.readdirSync(logsDir) + .filter(f => f.endsWith('.jsonl')) + .sort((a, b) => b.localeCompare(a)) + : [] + + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ files })) + return + } + + // GET /api/logs?file=...&limit=500 + const fileParam = url.searchParams.get('file') || '' + const safeName = path.basename(fileParam) + const limit = Number.parseInt(url.searchParams.get('limit') || '500', 10) + const lineLimit = Number.isFinite(limit) ? Math.max(1, Math.min(limit, 5000)) : 500 + const targetPath = path.join(logsDir, safeName) + + if (!fs.existsSync(targetPath)) { + res.writeHead(404, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ error: 'file not found' })) + return + } + + try { + const lines = fs.readFileSync(targetPath, 'utf-8') + .trim() + .split('\n') + .slice(-lineLimit) + const events = lines + .map((line) => { + try { return JSON.parse(line) } + catch { return null } + }) + .filter(Boolean) + + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ file: safeName, events, total: lines.length })) + } + catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ error: 'failed to read log file', message: (err as Error).message })) + } + } } diff --git a/services/minecraft/src/debug/web/app.js b/services/minecraft/src/debug/web/app.js index e2eb4487a..9280aefd3 100644 --- a/services/minecraft/src/debug/web/app.js +++ b/services/minecraft/src/debug/web/app.js @@ -412,11 +412,14 @@ class LogsPanel { this.autoScroll = true this.paused = false this.filter = { level: 'all', search: '' } + this.currentFile = '' this.elements = { container: document.getElementById('logs-container'), search: document.getElementById('log-search'), levelFilter: document.getElementById('log-level-filter'), autoScroll: document.getElementById('auto-scroll'), + fileSelect: document.getElementById('log-file-select'), + loadFile: document.getElementById('load-log-file'), statEvents: document.getElementById('stat-events'), } } @@ -439,6 +442,18 @@ class LogsPanel { this.autoScroll = e.target.checked }) + this.elements.loadFile.addEventListener('click', () => { + const file = this.elements.fileSelect.value + if (!file) { + this.currentFile = '' + this.reset() + return + } + this.loadPersistedLog(file) + }) + + this.refreshFileList() + this.renderThrottled = throttle(() => this.render(), CONFIG.UPDATE_THROTTLE) this.render() } @@ -462,6 +477,47 @@ class LogsPanel { this.render() } + async refreshFileList() { + try { + const res = await fetch('/api/logs') + const json = await res.json() + const files = json.files || [] + const select = this.elements.fileSelect + if (!select) + return + const current = select.value + select.innerHTML = '' + files.map(f => ``).join('') + if (files.includes(current)) + select.value = current + } + catch (err) { + console.error('Failed to load log files', err) + } + } + + async loadPersistedLog(file) { + try { + const res = await fetch(`/api/logs?file=${encodeURIComponent(file)}&limit=1000`) + if (!res.ok) { + console.error('Failed to fetch log file', await res.text()) + return + } + const json = await res.json() + const events = Array.isArray(json.events) ? json.events : [] + const logEntries = events + .filter(e => e.type === 'log') + .map(e => e.payload) + .filter(Boolean) + this.logs = logEntries + this.currentFile = file + this.elements.statEvents.textContent = this.logs.length + this.render() + } + catch (err) { + console.error('Failed to load persisted log', err) + } + } + clear() { this.logs = [] this.elements.statEvents.textContent = '0' @@ -556,8 +612,8 @@ class LLMPanel {
`).join('') } diff --git a/services/minecraft/src/debug/web/index.html b/services/minecraft/src/debug/web/index.html index 9649d80ae..8287929c0 100644 --- a/services/minecraft/src/debug/web/index.html +++ b/services/minecraft/src/debug/web/index.html @@ -138,6 +138,10 @@ + +