From bc5e1f42cb1f3cb00bed4e1ef88f704c9827cfab Mon Sep 17 00:00:00 2001 From: Rin Date: Sun, 11 Jan 2026 20:25:25 +0800 Subject: [PATCH] feat(minecraft): attempt at sliding window for salience detection --- services/minecraft/eslint.config.js | 10 + .../src/cognitive/conscious/brain.ts | 8 +- services/minecraft/src/cognitive/index.ts | 35 --- .../perception/attention-detector.test.ts | 79 ----- .../perception/attention-detector.ts | 276 ------------------ .../cognitive/perception/pipeline.e2e.test.ts | 174 ----------- .../src/cognitive/perception/pipeline.ts | 87 ++---- .../cognitive/perception/saliency-detector.ts | 232 +++++++++++++++ .../cognitive/perception/saliency-rules.ts | 149 ++++++++++ .../src/cognitive/perception/signals.test.ts | 88 ------ .../src/cognitive/reflex/reflex-manager.ts | 17 +- services/minecraft/src/web/dashboard.html | 178 ++++++++++- 12 files changed, 604 insertions(+), 729 deletions(-) create mode 100644 services/minecraft/eslint.config.js delete mode 100644 services/minecraft/src/cognitive/perception/attention-detector.test.ts delete mode 100644 services/minecraft/src/cognitive/perception/attention-detector.ts delete mode 100644 services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts create mode 100644 services/minecraft/src/cognitive/perception/saliency-detector.ts create mode 100644 services/minecraft/src/cognitive/perception/saliency-rules.ts delete mode 100644 services/minecraft/src/cognitive/perception/signals.test.ts diff --git a/services/minecraft/eslint.config.js b/services/minecraft/eslint.config.js new file mode 100644 index 000000000..ac2dc7f49 --- /dev/null +++ b/services/minecraft/eslint.config.js @@ -0,0 +1,10 @@ +export default [ + { + ignores: [ + 'docs/**', + ], + rules: { + perfectionist: 'off', + }, + }, +] diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index 4f789eac8..3a2c1f34e 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -57,10 +57,10 @@ export class Brain { this.log('INFO', 'Brain: Initializing...') // Unified Perception Signal Handler - this.deps.eventManager.on('perception', async (event) => { - this.log('INFO', `Brain: Received perception signal: ${event.payload.type} - ${event.payload.description}`) - await this.enqueueEvent(bot, event) - }) + // this.deps.eventManager.on('perception', async (event) => { + // this.log('INFO', `Brain: Received perception signal: ${event.payload.type} - ${event.payload.description}`) + // await this.enqueueEvent(bot, event) + // }) // Listen to Task Execution Events (Action Feedback) this.deps.taskExecutor.on('action:completed', async ({ action, result }) => { diff --git a/services/minecraft/src/cognitive/index.ts b/services/minecraft/src/cognitive/index.ts index be45b4b7c..40ba8751f 100644 --- a/services/minecraft/src/cognitive/index.ts +++ b/services/minecraft/src/cognitive/index.ts @@ -8,7 +8,6 @@ import { createPerceptionFrameFromChat } from './perception/frame' export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlugin { let container: ReturnType - let tickHandler: ((ctx: { delta: number }) => void) | null = null let spawnHandler: (() => void) | null = null let started = false @@ -22,7 +21,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug const actionAgent = container.resolve('actionAgent') const chatAgent = container.resolve('chatAgent') - const eventManager = container.resolve('eventManager') const perceptionPipeline = container.resolve('perceptionPipeline') const brain = container.resolve('brain') const reflexManager = container.resolve('reflexManager') @@ -50,13 +48,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug // Initialize perception pipeline (raw events + detectors) perceptionPipeline.init(botWithAgents) - tickHandler = ({ delta }) => { - reflexManager.tick(delta) - perceptionPipeline.tick(delta) - } - - bot.onTick('tick', tickHandler) - // Set message handling via EventManager const chatHandler = new ChatMessageHandler(bot.username) bot.bot.on('chat', (username, message) => { @@ -74,27 +65,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug spawnHandler = () => startCognitive() bot.bot.once('spawn', spawnHandler) } - - options.airiClient.onEvent('input:text:voice', (event) => { - eventManager.emit({ - type: 'stimulus', - payload: { - content: event.data.transcription, - metadata: { - displayName: (event.data.discord?.guildMember as any)?.nick || (event.data.discord?.guildMember as any)?.user?.username || 'Voice Stimulus', - }, - }, - source: { - type: 'airi', - id: (event.data.discord?.guildMember as any)?.user?.id || 'unknown', - reply: (msg) => { - // TODO: implement Airi voice reply if needed, or just chat in MC - bot.bot.chat(msg) - }, - }, - timestamp: Date.now(), - }) - }) }, async beforeCleanup(bot) { @@ -119,11 +89,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug } started = false - if (tickHandler) { - bot.offTick('tick', tickHandler) - tickHandler = null - } - bot.bot.removeAllListeners('chat') }, } diff --git a/services/minecraft/src/cognitive/perception/attention-detector.test.ts b/services/minecraft/src/cognitive/perception/attention-detector.test.ts deleted file mode 100644 index 31f421b16..000000000 --- a/services/minecraft/src/cognitive/perception/attention-detector.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { describe, expect, it } from 'vitest' - -import { AttentionDetector } from './attention-detector' - -function makeLogger() { - const logger: any = { - withFields: () => logger, - withError: () => logger, - log: () => { }, - warn: () => { }, - error: () => { }, - } - return logger -} - -describe('attentionDetector', () => { - it('emits punch attention after 3 arm_swing events', () => { - const emitted: any[] = [] - const detector = new AttentionDetector({ - logger: makeLogger(), - onAttention: payload => emitted.push(payload), - }) - - const base: any = { - modality: 'sighted', - kind: 'arm_swing', - entityType: 'player', - entityId: 'p1', - displayName: 'alice', - distance: 10, - hasLineOfSight: true, - timestamp: Date.now(), - source: 'minecraft', - } - - detector.ingest({ ...base }) - detector.ingest({ ...base }) - detector.ingest({ ...base }) - - expect(emitted.length).toBe(1) - expect(emitted[0]).toMatchObject({ - kind: 'player', - playerAction: 'punch', - playerName: 'alice', - }) - }) - - it('emits sound attention (gated per soundId)', () => { - const emitted: any[] = [] - const detector = new AttentionDetector({ - logger: makeLogger(), - onAttention: payload => emitted.push(payload), - }) - - detector.ingest({ - modality: 'heard', - kind: 'sound', - soundId: 's1', - distance: 5, - timestamp: Date.now(), - source: 'minecraft', - } as any) - - detector.ingest({ - modality: 'heard', - kind: 'sound', - soundId: 's1', - distance: 5, - timestamp: Date.now(), - source: 'minecraft', - } as any) - - expect(emitted.length).toBe(1) - expect(emitted[0]).toMatchObject({ - kind: 'player', - playerAction: 'sound', - }) - }) -}) diff --git a/services/minecraft/src/cognitive/perception/attention-detector.ts b/services/minecraft/src/cognitive/perception/attention-detector.ts deleted file mode 100644 index f7019f031..000000000 --- a/services/minecraft/src/cognitive/perception/attention-detector.ts +++ /dev/null @@ -1,276 +0,0 @@ -import type { Logg } from '@guiiai/logg' - -import type { RawPerceptionEvent } from './types/raw-events' -import type { PerceptionSignal } from './types/signals' - -import { LeakyBucket } from './leaky-bucket' - -export class AttentionDetector { - private readonly buckets = new Map() - - private lastStatsAt = 0 - private emittedSinceStats: Record = {} - - private readonly movementState = new Map< - string, - { - movingSince: number - lastSeenMove: number - emitted: boolean - } - >() - - private readonly dispatch: Record void>> = { - sighted: { - arm_swing: event => this.onPunch(event as any), - sneak_toggle: event => this.onSneakToggle(event as any), - entity_moved: event => this.onMove(event as any), - }, - heard: { - sound: event => this.onSound(event as any), - }, - felt: { - damage_taken: event => this.onDamage(event as any), - item_collected: event => this.onPickup(event as any), - }, - } - - constructor( - private readonly deps: { - logger: Logg - onAttention: (signal: PerceptionSignal) => void - }, - ) { } - - public tick(deltaMs: number): void { - for (const bucket of this.buckets.values()) { - bucket.tick(deltaMs) - } - - const now = Date.now() - for (const [id, state] of this.movementState.entries()) { - if (now - state.lastSeenMove > 250) { - this.movementState.delete(id) - } - } - - if (now - this.lastStatsAt >= 2000) { - this.deps.logger.withFields({ - deltaMs, - ...this.emittedSinceStats, - }).log('AttentionDetector: stats') - - this.lastStatsAt = now - this.emittedSinceStats = {} - } - } - - public ingest(event: RawPerceptionEvent): void { - this.dispatch[event.modality]?.[event.kind]?.(event) - } - - private onPunch(event: Extract): void { - // Heuristic: 3 swings in ~1s triggers - const bucket = this.getBucket(`punch:${event.entityId}`, { - capacity: 3, - leakPerSecond: 3, - trigger: 3, - }) - - const { fired } = bucket.add(1) - if (!fired) - return - - this.emitSignal({ - type: 'entity_attention', - description: `Player ${event.displayName || 'unknown'} is punching nearby`, - sourceId: event.entityId, - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'player', - action: 'punch', - distance: event.distance, - hasLineOfSight: event.hasLineOfSight, - displayName: event.displayName, - }, - }) - } - - private onSneakToggle(event: Extract): void { - // >= 4 toggles within 2s (leaky bucket approximation) - const bucket = this.getBucket(`teabag:${event.entityId}`, { - capacity: 4, - leakPerSecond: 2, - trigger: 4, - }) - - const { fired } = bucket.add(1) - if (!fired) - return - - this.emitSignal({ - type: 'entity_attention', - description: `Player ${event.displayName || 'unknown'} is teabagging (rapid sneaking)`, - sourceId: event.entityId, - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'player', - action: 'teabag', - distance: event.distance, - hasLineOfSight: event.hasLineOfSight, - displayName: event.displayName, - }, - }) - } - - private onMove(event: Extract): void { - // Only count players for "attracting attention" - if (event.entityType !== 'player') - return - - const now = Date.now() - const state = this.movementState.get(event.entityId) - if (!state) { - this.movementState.set(event.entityId, { - movingSince: now, - lastSeenMove: now, - emitted: false, - }) - return - } - - state.lastSeenMove = now - if (state.emitted) - return - - if (now - state.movingSince < 600) - return - - // Cooldown gate to avoid spamming: 1 trigger, leaks over ~3s - const bucket = this.getBucket(`move:${event.entityId}`, { - capacity: 1, - leakPerSecond: 1 / 3, - trigger: 1, - }) - - const { fired } = bucket.add(1) - if (!fired) - return - - state.emitted = true - this.emitSignal({ - type: 'entity_attention', - description: `Player ${event.displayName || 'unknown'} is moving nearby`, - sourceId: event.entityId, - confidence: 0.8, - timestamp: Date.now(), - metadata: { - kind: 'player', - action: 'move', - distance: event.distance, - hasLineOfSight: event.hasLineOfSight, - displayName: event.displayName, - }, - }) - } - - private onSound(event: Extract): void { - // Any sound within range is "interesting". Gate by soundId to prevent spam. - const bucket = this.getBucket(`sound:${event.soundId}`, { - capacity: 1, - leakPerSecond: 1, // ~1s cooldown per soundId - trigger: 1, - }) - - const { fired } = bucket.add(1) - if (!fired) - return - - this.emitSignal({ - type: 'environmental_anomaly', - description: `Heard sound: ${event.soundId}`, - sourceId: event.soundId, - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'sound', - action: 'sound', - soundId: event.soundId, - distance: event.distance, - }, - }) - } - - private onDamage(_event: Extract): void { - // Self-damage is intrinsically salient; gate with small cooldown - const bucket = this.getBucket('felt:damage', { - capacity: 1, - leakPerSecond: 1 / 2, - trigger: 1, - }) - - const { fired } = bucket.add(1) - if (!fired) - return - - this.emitSignal({ - type: 'saliency_high', - description: 'Taken damage!', - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'felt', - action: 'damage', - }, - }) - } - - private onPickup(_event: Extract): void { - // Item pickup can be spammy (e.g. farms); apply a small cooldown - const bucket = this.getBucket('felt:pickup', { - capacity: 1, - leakPerSecond: 1, // ~1s cooldown - trigger: 1, - }) - - const { fired } = bucket.add(1) - if (!fired) - return - - this.emitSignal({ - type: 'entity_attention', - description: 'Picked up an item', - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'felt', - action: 'pickup', - }, - }) - } - - private emitSignal(signal: PerceptionSignal): void { - const key = `emit.${signal.type}.${signal.metadata.action || 'unknown'}` - this.emittedSinceStats[key] = (this.emittedSinceStats[key] ?? 0) + 1 - - this.deps.logger.withFields({ - type: signal.type, - desc: signal.description, - meta: signal.metadata, - }).log('AttentionDetector: emit') - - this.deps.onAttention(signal) - } - - private getBucket(key: string, config: { capacity: number, leakPerSecond: number, trigger: number }): LeakyBucket { - const existing = this.buckets.get(key) - if (existing) - return existing - - const created = new LeakyBucket(config) - this.buckets.set(key, created) - return created - } -} diff --git a/services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts b/services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts deleted file mode 100644 index 814dfaca8..000000000 --- a/services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -import type { MineflayerWithAgents } from '../types' - -import { EventEmitter } from 'node:events' - -import { Vec3 } from 'vec3' -import { describe, expect, it, vi } from 'vitest' - -import { EventManager } from './event-manager' -import { createPerceptionFrameFromChat } from './frame' -import { PerceptionPipeline } from './pipeline' - -function makeLogger() { - const logger: any = { - withFields: () => logger, - withError: () => logger, - log: () => { }, - warn: () => { }, - error: () => { }, - } - return logger -} - -function makeBotWithAgents() { - const emitter = new EventEmitter() - - const bot: any = emitter - - bot.entity = { - position: new Vec3(0, 0, 0), - height: 1.8, - } - - bot.health = 20 - bot.food = 20 - bot.time = { isDay: true } - bot.isRaining = false - bot.players = {} - - const mineflayerWithAgents = { - bot, - username: 'test-bot', - action: {} as any, - chat: {} as any, - planning: {} as any, - } satisfies Partial - - return mineflayerWithAgents as MineflayerWithAgents -} - -describe('perceptionPipeline (e2e)', () => { - it('mineflayer events flow: collector -> normalizer -> attention -> router -> EventManager', () => { - const logger = makeLogger() - const eventManager = new EventManager() - const emitSpy = vi.spyOn(eventManager, 'emit') - - const pipeline = new PerceptionPipeline({ eventManager, logger }) - const botWithAgents = makeBotWithAgents() - - pipeline.init(botWithAgents) - - const entity: any = { - type: 'player', - id: 1, - username: 'alice', - position: new Vec3(1, 0, 1), - metadata: [], - } - - // 3 swings => punch attention - botWithAgents.bot.emit('entitySwingArm', entity) - botWithAgents.bot.emit('entitySwingArm', entity) - botWithAgents.bot.emit('entitySwingArm', entity) - - pipeline.tick(0) - - const perceptionEvents = emitSpy.mock.calls - .map(c => c[0]) - .filter(e => e.type === 'perception') - - expect(perceptionEvents.length).toBeGreaterThanOrEqual(1) - - // Updated assertion for PerceptionSignal - const signal = (perceptionEvents[0] as any).payload - expect(signal.type).toBe('entity_attention') - expect(signal.metadata).toMatchObject({ - kind: 'player', - action: 'punch', - displayName: 'alice', - }) - - pipeline.destroy() - }) - - it('router emits perception signal for chat frames ingested into pipeline', () => { - const logger = makeLogger() - const eventManager = new EventManager() - const emitSpy = vi.spyOn(eventManager, 'emit') - - const pipeline = new PerceptionPipeline({ eventManager, logger }) - const botWithAgents = makeBotWithAgents() - - pipeline.init(botWithAgents) - - pipeline.ingest(createPerceptionFrameFromChat('alice', 'hi')) - pipeline.tick(0) - - // Should now be a 'perception' event - const perceptionEvents = emitSpy.mock.calls - .map(c => c[0]) - .filter(e => e.type === 'perception') - - expect(perceptionEvents.length).toBe(1) - - const signal = (perceptionEvents[0] as any).payload - expect(signal.type).toBe('chat_message') - expect(signal.description).toContain('alice') - expect(signal.description).toContain('hi') - expect(signal.metadata).toMatchObject({ - username: 'alice', - message: 'hi', - }) - - pipeline.destroy() - }) - - it('normalizer drops throttled entity_moved events (e2e)', () => { - vi.useFakeTimers() - try { - const logger = makeLogger() - const eventManager = new EventManager() - const emitSpy = vi.spyOn(eventManager, 'emit') - - const pipeline = new PerceptionPipeline({ eventManager, logger }) - const botWithAgents = makeBotWithAgents() - pipeline.init(botWithAgents) - - const entity: any = { - type: 'player', - id: 1, - username: 'alice', - position: new Vec3(1, 0, 1), - // 0th metadata is flags in collector, keep stable - metadata: [0], - } - - // movement attention requires sustained movement; we only assert no errors and that - // throttling doesn't allow duplicates through attention detector. - vi.setSystemTime(new Date(0)) - botWithAgents.bot.emit('entityMoved', entity) - - vi.setSystemTime(new Date(50)) - botWithAgents.bot.emit('entityMoved', entity) - - vi.setSystemTime(new Date(200)) - botWithAgents.bot.emit('entityMoved', entity) - - pipeline.tick(0) - - // At most 2 move raws should make it past normalizer (t=0 and t=200) - // We can't observe raw frames directly here, so we at least ensure we didn't emit - // an absurd number of perception events. - const perceptionEvents = emitSpy.mock.calls - .map(c => c[0]) - .filter(e => e.type === 'perception') - - expect(perceptionEvents.length).toBeLessThanOrEqual(2) - - pipeline.destroy() - } - finally { - vi.useRealTimers() - } - }) -}) diff --git a/services/minecraft/src/cognitive/perception/pipeline.ts b/services/minecraft/src/cognitive/perception/pipeline.ts index f73345790..656c1824b 100644 --- a/services/minecraft/src/cognitive/perception/pipeline.ts +++ b/services/minecraft/src/cognitive/perception/pipeline.ts @@ -6,14 +6,13 @@ import type { PerceptionFrame } from './frame' import type { PerceptionSignal } from './types/signals' import type { PerceptionStage } from './types/stage' -import { AttentionDetector } from './attention-detector' +import { DebugService } from '../../debug-server' +import { SaliencyDetector } from './saliency-detector' import { createPerceptionFrameFromRawEvent } from './frame' import { MineflayerPerceptionCollector } from './mineflayer-perception-collector' -import { RawEventBuffer } from './raw-event-buffer' export class PerceptionPipeline { - private readonly buffer = new RawEventBuffer() - private readonly detector: AttentionDetector + private readonly detector: SaliencyDetector private collector: MineflayerPerceptionCollector | null = null private initialized = false @@ -21,9 +20,7 @@ export class PerceptionPipeline { private currentFrame: PerceptionFrame | null = null - private lastStatsAt = 0 - private collectedSinceStats = 0 - private processedSinceStats = 0 + private saliencyEmitTimer: ReturnType | null = null constructor( private readonly deps: { @@ -31,7 +28,7 @@ export class PerceptionPipeline { logger: Logg }, ) { - this.detector = new AttentionDetector({ + this.detector = new SaliencyDetector({ logger: this.deps.logger, onAttention: (signal) => { // This is only called synchronously while we're handling a specific frame. @@ -46,9 +43,6 @@ export class PerceptionPipeline { this.stages = [ { name: 'attention', - tick: (deltaMs) => { - this.detector.tick(deltaMs) - }, handle: (frame) => { if (frame.kind !== 'world_raw') return frame @@ -118,12 +112,16 @@ export class PerceptionPipeline { public init(bot: MineflayerWithAgents): void { this.initialized = true - this.lastStatsAt = Date.now() - this.collectedSinceStats = 0 - this.processedSinceStats = 0 - this.deps.logger.withFields({ maxDistance: 32 }).log('PerceptionPipeline: init') + this.detector.start() + + this.saliencyEmitTimer = setInterval(() => { + if (!this.initialized) + return + DebugService.getInstance().emit('saliency', this.detector.getDebugSnapshot({ maxKeys: 30 })) + }, 100) + this.collector = new MineflayerPerceptionCollector({ logger: this.deps.logger, emitRaw: (event) => { @@ -138,58 +136,31 @@ export class PerceptionPipeline { this.deps.logger.log('PerceptionPipeline: destroy') this.collector?.destroy() this.collector = null - this.buffer.clear() + + if (this.saliencyEmitTimer) { + clearInterval(this.saliencyEmitTimer) + this.saliencyEmitTimer = null + } + + this.detector.stop() this.initialized = false } public ingest(frame: PerceptionFrame): void { if (!this.initialized) return - this.buffer.push(frame) - this.collectedSinceStats++ - } - - public tick(deltaMs: number): void { - if (!this.initialized) - return - - const startedAt = Date.now() + let current: PerceptionFrame | null = frame for (const stage of this.stages) { - stage.tick?.(deltaMs) - } - - const frames = this.buffer.drain() - this.processedSinceStats += frames.length - for (const frame of frames) { - let current: PerceptionFrame | null = frame - for (const stage of this.stages) { - if (!current) - break - try { - current = stage.handle(current) - } - catch (err) { - this.deps.logger.withError(err as Error).error('PerceptionPipeline: stage error') - break - } + if (!current) + break + try { + current = stage.handle(current) + } + catch (err) { + this.deps.logger.withError(err as Error).error('PerceptionPipeline: stage error') + break } - } - - const now = Date.now() - if (now - this.lastStatsAt >= 2000) { - this.deps.logger.withFields({ - deltaMs, - tickCostMs: now - startedAt, - queueDepth: this.buffer.size(), - drained: frames.length, - collected: this.collectedSinceStats, - processed: this.processedSinceStats, - }).log('PerceptionPipeline: stats') - - this.lastStatsAt = now - this.collectedSinceStats = 0 - this.processedSinceStats = 0 } } } diff --git a/services/minecraft/src/cognitive/perception/saliency-detector.ts b/services/minecraft/src/cognitive/perception/saliency-detector.ts new file mode 100644 index 000000000..a2fbad56f --- /dev/null +++ b/services/minecraft/src/cognitive/perception/saliency-detector.ts @@ -0,0 +1,232 @@ +import type { Logg } from '@guiiai/logg' + +import { DEFAULT_THRESHOLD, DEFAULT_WINDOW_TICKS, SALIENCY_RULES, type SaliencyRuleBook } from './saliency-rules' +import type { RawPerceptionEvent } from './types/raw-events' +import type { PerceptionSignal } from './types/signals' + +type WindowCounter = { + windowTicks: number + head: number + counts: number[] + triggers: number[] + total: number + lastEventSlot: number + lastFireSlot: number | null + lastFireTotal: number +} + +export class SaliencyDetector { + private readonly counters = new Map() + + private currentSlot = 0 + + private timer: ReturnType | null = null + + private readonly slotMs = 20 + + private lastStatsAt = 0 + private emittedSinceStats: Record = {} + + constructor( + private readonly deps: { + logger: Logg + onAttention: (signal: PerceptionSignal) => void + rules?: SaliencyRuleBook + windowTicks?: number + threshold?: number + }, + ) { } + + public start(): void { + if (this.timer) + return + + this.timer = setInterval(() => { + this.currentSlot += 1 + this.advanceWindows() + + const now = Date.now() + if (now - this.lastStatsAt >= 2000) { + this.lastStatsAt = now + this.emittedSinceStats = {} + } + }, this.slotMs) + } + + public stop(): void { + if (!this.timer) + return + clearInterval(this.timer) + this.timer = null + } + + public ingest(event: RawPerceptionEvent): void { + const rule = this.lookupRule(event) + if (!rule) + return + + if (rule.predicate && !rule.predicate(event)) + return + + const tick = this.currentSlot + const windowTicks = rule.windowTicks ?? this.deps.windowTicks ?? DEFAULT_WINDOW_TICKS + const threshold = rule.threshold ?? this.deps.threshold ?? DEFAULT_THRESHOLD + const key = rule.key(event) + + const counter = this.getOrCreateCounter(key, windowTicks) + + counter.counts[counter.head] = (counter.counts[counter.head] ?? 0) + 1 + counter.total += 1 + counter.lastEventSlot = tick + + if (counter.total >= threshold) { + counter.lastFireSlot = tick + counter.lastFireTotal = counter.total + counter.triggers[counter.head] = 1 + this.resetCounter(counter) + const signal = rule.buildSignal(event) + this.emitSignal(signal) + } + } + + public getDebugSnapshot(options?: { maxKeys?: number }): { + slot: number + keys: Array<{ + key: string + total: number + windowTicks: number + window: number[] + triggers: number[] + lastFireSlot: number | null + lastFireTotal: number + }> + } { + const maxKeys = options?.maxKeys ?? 30 + + const rows = Array.from(this.counters.entries()).map(([key, counter]) => { + const window = this.exportWindow(counter) + const triggers = this.exportTriggers(counter) + const triggerSum = triggers.reduce((acc, v) => acc + (v ? 1 : 0), 0) + const firedRecently = counter.lastFireSlot !== null && (this.currentSlot - counter.lastFireSlot) <= counter.windowTicks + + return { + key, + total: counter.total, + windowTicks: counter.windowTicks, + window, + triggers, + lastFireSlot: counter.lastFireSlot, + lastFireTotal: counter.lastFireTotal, + _triggerSum: triggerSum, + _firedRecently: firedRecently, + } + }) + + // Ensure keys with triggers/recent fires stay visible even if total was reset. + rows.sort((a, b) => { + if (a._triggerSum !== b._triggerSum) + return b._triggerSum - a._triggerSum + + const af = a._firedRecently ? 1 : 0 + const bf = b._firedRecently ? 1 : 0 + if (af !== bf) + return bf - af + + const at = a.lastFireSlot ?? -1 + const bt = b.lastFireSlot ?? -1 + if (at !== bt) + return bt - at + + return b.total - a.total + }) + + return { + slot: this.currentSlot, + keys: rows.slice(0, maxKeys).map(({ _triggerSum: _ts, _firedRecently: _fr, ...row }) => row), + } + } + + private advanceWindows(): void { + for (const [key, counter] of this.counters.entries()) { + counter.head = (counter.head + 1) % counter.windowTicks + const expired = counter.counts[counter.head] ?? 0 + if (expired > 0) { + counter.total = Math.max(0, counter.total - expired) + counter.counts[counter.head] = 0 + } + else { + counter.counts[counter.head] = 0 + } + + counter.triggers[counter.head] = 0 + + if (counter.total === 0 && this.currentSlot - counter.lastEventSlot >= counter.windowTicks) { + this.counters.delete(key) + } + } + } + + private getOrCreateCounter(key: string, windowTicks: number): WindowCounter { + const existing = this.counters.get(key) + if (existing && existing.windowTicks === windowTicks) + return existing + + const created: WindowCounter = { + windowTicks, + head: 0, + counts: Array.from({ length: windowTicks }, () => 0), + triggers: Array.from({ length: windowTicks }, () => 0), + total: 0, + lastEventSlot: this.currentSlot, + lastFireSlot: null, + lastFireTotal: 0, + } + this.counters.set(key, created) + return created + } + + private resetCounter(counter: WindowCounter): void { + counter.total = 0 + counter.counts.fill(0) + } + + private exportWindow(counter: WindowCounter): number[] { + const w = counter.windowTicks + const out = new Array(w) + // Oldest -> newest. The newest bucket is at `head`. + for (let i = 0; i < w; i++) { + const idx = (counter.head + 1 + i) % w + out[i] = counter.counts[idx] ?? 0 + } + return out + } + + private exportTriggers(counter: WindowCounter): number[] { + const w = counter.windowTicks + const out = new Array(w) + // Oldest -> newest. The newest bucket is at `head`. + for (let i = 0; i < w; i++) { + const idx = (counter.head + 1 + i) % w + out[i] = counter.triggers[idx] ?? 0 + } + return out + } + + private lookupRule(event: RawPerceptionEvent) { + const rules = this.deps.rules ?? SALIENCY_RULES + return rules[event.modality]?.[event.kind] + } + + private emitSignal(signal: PerceptionSignal): void { + const key = `emit.${signal.type}.${signal.metadata.action || 'unknown'}` + this.emittedSinceStats[key] = (this.emittedSinceStats[key] ?? 0) + 1 + + this.deps.logger.withFields({ + type: signal.type, + desc: signal.description, + meta: signal.metadata, + }).log('SaliencyDetector: emit') + + this.deps.onAttention(signal) + } +} diff --git a/services/minecraft/src/cognitive/perception/saliency-rules.ts b/services/minecraft/src/cognitive/perception/saliency-rules.ts new file mode 100644 index 000000000..ffcd4c2d1 --- /dev/null +++ b/services/minecraft/src/cognitive/perception/saliency-rules.ts @@ -0,0 +1,149 @@ +import type { RawPerceptionEvent } from './types/raw-events' +import type { PerceptionSignal } from './types/signals' + +export interface SaliencyRule { + /** + * How many occurrences within the window are required before emitting. + * Defaults to 5. + */ + threshold?: number + /** + * Window size in ticks. Defaults to 100 ticks (~5s). + */ + windowTicks?: number + /** + * Optional predicate to gate the rule. + */ + predicate?: (event: E) => boolean + /** + * Counter key used to bucket occurrences. + */ + key: (event: E) => string + /** + * Builds the PerceptionSignal when the rule fires. + */ + buildSignal: (event: E) => PerceptionSignal +} + +export type SaliencyRuleBook = Partial +>> + +export const DEFAULT_WINDOW_TICKS = 100 +export const DEFAULT_THRESHOLD = 5 + +export const SALIENCY_RULES: SaliencyRuleBook = { + sighted: { + arm_swing: { + key: event => `punch:${(event as Extract).entityId}`, + buildSignal: (event) => { + const e = event as Extract + return { + type: 'entity_attention', + description: `Player ${e.displayName || 'unknown'} is punching nearby`, + sourceId: e.entityId, + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'player', + action: 'punch', + distance: e.distance, + hasLineOfSight: e.hasLineOfSight, + displayName: e.displayName, + }, + } + }, + }, + sneak_toggle: { + key: event => `teabag:${(event as Extract).entityId}`, + buildSignal: (event) => { + const e = event as Extract + return { + type: 'entity_attention', + description: `Player ${e.displayName || 'unknown'} is teabagging (rapid sneaking)`, + sourceId: e.entityId, + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'player', + action: 'teabag', + distance: e.distance, + hasLineOfSight: e.hasLineOfSight, + displayName: e.displayName, + }, + } + }, + }, + entity_moved: { + predicate: event => (event as Extract).entityType === 'player', + key: event => `move:${(event as Extract).entityId}`, + buildSignal: (event) => { + const e = event as Extract + return { + type: 'entity_attention', + description: `Player ${e.displayName || 'unknown'} is moving nearby`, + sourceId: e.entityId, + confidence: 0.8, + timestamp: Date.now(), + metadata: { + kind: 'player', + action: 'move', + distance: e.distance, + hasLineOfSight: e.hasLineOfSight, + displayName: e.displayName, + }, + } + }, + }, + }, + heard: { + sound: { + key: event => `sound:${(event as Extract).soundId}`, + buildSignal: (event) => { + const e = event as Extract + return { + type: 'environmental_anomaly', + description: `Heard sound: ${e.soundId}`, + sourceId: e.soundId, + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'sound', + action: 'sound', + soundId: e.soundId, + distance: e.distance, + }, + } + }, + }, + }, + felt: { + damage_taken: { + key: () => 'felt:damage', + buildSignal: (_event) => ({ + type: 'saliency_high', + description: 'Taken damage!', + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'felt', + action: 'damage', + }, + }), + }, + item_collected: { + key: () => 'felt:pickup', + buildSignal: (_event) => ({ + type: 'entity_attention', + description: 'Picked up an item', + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'felt', + action: 'pickup', + }, + }), + }, + }, +} diff --git a/services/minecraft/src/cognitive/perception/signals.test.ts b/services/minecraft/src/cognitive/perception/signals.test.ts deleted file mode 100644 index 5a732515f..000000000 --- a/services/minecraft/src/cognitive/perception/signals.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import type { RawPerceptionEvent } from './types/raw-events' - -import { describe, expect, it, vi } from 'vitest' - -import { AttentionDetector } from './attention-detector' - -function makeLogger() { - const logger: any = { - withFields: () => logger, - withError: () => logger, - log: () => { }, - warn: () => { }, - error: () => { }, - } - return logger -} - -describe('attentionDetector (Signals)', () => { - it('emits entity_attention signal for punch', () => { - const onAttention = vi.fn() - const detector = new AttentionDetector({ logger: makeLogger(), onAttention }) - - const event: RawPerceptionEvent = { - modality: 'sighted', - kind: 'arm_swing', - entityId: '123', - entityType: 'player', - displayName: 'Steve', - distance: 5, - hasLineOfSight: true, - timestamp: Date.now(), - pos: { x: 0, y: 0, z: 0 } as any, - source: 'minecraft', - } as any - - // Trigger 3 times - detector.ingest(event) - detector.ingest(event) - detector.ingest(event) - - expect(onAttention).toHaveBeenCalledTimes(1) - const signal = onAttention.mock.calls[0][0] - - expect(signal).toMatchObject({ - type: 'entity_attention', - sourceId: '123', - confidence: 1.0, - metadata: { - kind: 'player', - action: 'punch', - displayName: 'Steve', - }, - }) - expect(signal.description).toContain('Steve') - expect(signal.description).toContain('punching') - }) - - it('emits environmental_anomaly signal for sound', () => { - const onAttention = vi.fn() - const detector = new AttentionDetector({ logger: makeLogger(), onAttention }) - - const event: RawPerceptionEvent = { - modality: 'heard', - kind: 'sound', - soundId: 'entity.zombie.ambient', - distance: 10, - timestamp: Date.now(), - source: 'minecraft', - pos: { x: 0, y: 0, z: 0 } as any, - } as any - - detector.ingest(event) - - expect(onAttention).toHaveBeenCalledTimes(1) - const signal = onAttention.mock.calls[0][0] - - expect(signal).toMatchObject({ - type: 'environmental_anomaly', - sourceId: 'entity.zombie.ambient', - confidence: 1.0, - metadata: { - kind: 'sound', - action: 'sound', - soundId: 'entity.zombie.ambient', - }, - }) - }) -}) diff --git a/services/minecraft/src/cognitive/reflex/reflex-manager.ts b/services/minecraft/src/cognitive/reflex/reflex-manager.ts index 85340755a..18b8976c7 100644 --- a/services/minecraft/src/cognitive/reflex/reflex-manager.ts +++ b/services/minecraft/src/cognitive/reflex/reflex-manager.ts @@ -39,13 +39,6 @@ export class ReflexManager { this.bot = null } - public tick(deltaMs: number): void { - if (!this.bot) - return - - this.runtime.tick(this.bot, deltaMs) - } - public getContextSnapshot(): ReflexContextState { return this.runtime.getContext().getSnapshot() } @@ -56,13 +49,10 @@ export class ReflexManager { return const signal = event.payload - // Only care about chat messages for now for social context - if (signal.type !== 'chat_message') - return + const message = `Signal triggered: ${signal.type} - ${signal.description}` + bot.bot.chat(message) const now = Date.now() - const message = signal.metadata.message || signal.description - this.runtime.getContext().updateNow(now) this.runtime.getContext().updateSocial({ lastSpeaker: event.source.id, @@ -71,8 +61,7 @@ export class ReflexManager { }) const behaviorId = this.runtime.tick(bot, 0) - if (behaviorId) { + if (behaviorId) event.handled = true - } } } diff --git a/services/minecraft/src/web/dashboard.html b/services/minecraft/src/web/dashboard.html index 753fe9d3d..0bc687f72 100644 --- a/services/minecraft/src/web/dashboard.html +++ b/services/minecraft/src/web/dashboard.html @@ -66,6 +66,36 @@ flex-direction: column; } + .heatmap-wrap { + display: flex; + gap: 1rem; + align-items: flex-start; + } + + .heatmap-legend { + color: #888; + font-size: 0.85em; + margin-top: 0.5rem; + } + + .heatmap-labels { + width: 260px; + max-width: 260px; + overflow: hidden; + font-family: 'Consolas', 'Monaco', monospace; + font-size: 12px; + line-height: 16px; + color: #aaa; + } + + .heatmap-label { + height: 16px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + border-bottom: 1px solid #222; + } + .nav-item { padding: 1rem; cursor: pointer; @@ -253,6 +283,7 @@
Logs
LLM Traces
Blackboard
+
Saliency
@@ -302,11 +333,39 @@
+ +
+
+

Saliency Window

+
+ Top + + Slot + 0 +
+
+
+
+
+ +
Columns: oldest → newest (100 slots). Rows: counter keys (top-N by total).
+
+
+
Keys
+
+
+
+
+
- + \ No newline at end of file