diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index 4cc290406..4f789eac8 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -4,8 +4,9 @@ import type { Neuri } from 'neuri' import type { TaskExecutor } from '../action/task-executor' import type { ActionInstruction } from '../action/types' import type { EventManager } from '../perception/event-manager' -import type { BotEvent, MineflayerWithAgents, StimulusPayload } from '../types' +import type { PerceptionSignal } from '../perception/types/signals' import type { ReflexManager } from '../reflex/reflex-manager' +import type { BotEvent, MineflayerWithAgents } from '../types' import { system, user } from 'neuri/openai' @@ -55,13 +56,9 @@ export class Brain { public init(bot: MineflayerWithAgents): void { this.log('INFO', 'Brain: Initializing...') - // We treat these as "Sensory Inputs" that trigger the Cognitive Cycle - this.deps.eventManager.on('stimulus', async (event) => { - if (event.handled) { - this.log('INFO', `Brain: Stimulus from ${event.source.id} already handled by reflex, ignoring.`) - return - } - this.log('INFO', `Brain: Received stimulus from ${event.source.id}: ${event.payload.content}`) + // Unified Perception Signal Handler + this.deps.eventManager.on('perception', async (event) => { + this.log('INFO', `Brain: Received perception signal: ${event.payload.type} - ${event.payload.description}`) await this.enqueueEvent(bot, event) }) @@ -140,8 +137,11 @@ export class Brain { private contextFromEvent(event: BotEvent): string { switch (event.type) { - case 'stimulus': - return `${event.source.type} stimulus from ${event.source.id}: "${event.payload.content}"` + case 'perception': { + const signal = event.payload as PerceptionSignal + const sourceInfo = signal.sourceId ? ` (source: ${signal.sourceId})` : '' + return `Perception [${signal.type}]${sourceInfo}: ${signal.description}` + } case 'feedback': { const { status, result, error } = event.payload return `Internal Feedback: ${status}. Result: ${JSON.stringify(result || error)}` diff --git a/services/minecraft/src/cognitive/perception/attention-detector.ts b/services/minecraft/src/cognitive/perception/attention-detector.ts index 139516b20..f7019f031 100644 --- a/services/minecraft/src/cognitive/perception/attention-detector.ts +++ b/services/minecraft/src/cognitive/perception/attention-detector.ts @@ -1,27 +1,10 @@ import type { Logg } from '@guiiai/logg' import type { RawPerceptionEvent } from './types/raw-events' +import type { PerceptionSignal } from './types/signals' import { LeakyBucket } from './leaky-bucket' -export interface PlayerAttentionEventPayload { - kind: 'player' - playerName?: string - hasLineOfSight?: boolean - distance?: number - playerAction: 'move' | 'punch' | 'teabag' | 'sound' | 'damage' | 'pickup' -} - -export interface MobAttentionEventPayload { - kind: 'mob' - mobName?: string - hasLineOfSight?: boolean - distance?: number - mobAction: string -} - -export type AttentionEventPayload = PlayerAttentionEventPayload | MobAttentionEventPayload - export class AttentionDetector { private readonly buckets = new Map() @@ -55,7 +38,7 @@ export class AttentionDetector { constructor( private readonly deps: { logger: Logg - onAttention: (payload: AttentionEventPayload) => void + onAttention: (signal: PerceptionSignal) => void }, ) { } @@ -98,12 +81,19 @@ export class AttentionDetector { if (!fired) return - this.emitAttention({ - kind: 'player', - playerName: event.displayName, - hasLineOfSight: event.hasLineOfSight, - distance: event.distance, - playerAction: 'punch', + this.emitSignal({ + type: 'entity_attention', + description: `Player ${event.displayName || 'unknown'} is punching nearby`, + sourceId: event.entityId, + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'player', + action: 'punch', + distance: event.distance, + hasLineOfSight: event.hasLineOfSight, + displayName: event.displayName, + }, }) } @@ -119,12 +109,19 @@ export class AttentionDetector { if (!fired) return - this.emitAttention({ - kind: 'player', - playerName: event.displayName, - hasLineOfSight: event.hasLineOfSight, - distance: event.distance, - playerAction: 'teabag', + this.emitSignal({ + type: 'entity_attention', + description: `Player ${event.displayName || 'unknown'} is teabagging (rapid sneaking)`, + sourceId: event.entityId, + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'player', + action: 'teabag', + distance: event.distance, + hasLineOfSight: event.hasLineOfSight, + displayName: event.displayName, + }, }) } @@ -163,12 +160,19 @@ export class AttentionDetector { return state.emitted = true - this.emitAttention({ - kind: 'player', - playerName: event.displayName, - hasLineOfSight: event.hasLineOfSight, - distance: event.distance, - playerAction: 'move', + this.emitSignal({ + type: 'entity_attention', + description: `Player ${event.displayName || 'unknown'} is moving nearby`, + sourceId: event.entityId, + confidence: 0.8, + timestamp: Date.now(), + metadata: { + kind: 'player', + action: 'move', + distance: event.distance, + hasLineOfSight: event.hasLineOfSight, + displayName: event.displayName, + }, }) } @@ -184,10 +188,18 @@ export class AttentionDetector { if (!fired) return - this.emitAttention({ - kind: 'player', - playerAction: 'sound', - distance: event.distance, + this.emitSignal({ + type: 'environmental_anomaly', + description: `Heard sound: ${event.soundId}`, + sourceId: event.soundId, + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'sound', + action: 'sound', + soundId: event.soundId, + distance: event.distance, + }, }) } @@ -203,9 +215,15 @@ export class AttentionDetector { if (!fired) return - this.emitAttention({ - kind: 'player', - playerAction: 'damage', + this.emitSignal({ + type: 'saliency_high', + description: 'Taken damage!', + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'felt', + action: 'damage', + }, }) } @@ -221,38 +239,29 @@ export class AttentionDetector { if (!fired) return - this.emitAttention({ - kind: 'player', - playerAction: 'pickup', + this.emitSignal({ + type: 'entity_attention', + description: 'Picked up an item', + confidence: 1.0, + timestamp: Date.now(), + metadata: { + kind: 'felt', + action: 'pickup', + }, }) } - private emitAttention(payload: AttentionEventPayload): void { - const key = payload.kind === 'player' - ? `emit.player.${payload.playerAction}` - : 'emit.mob' + private emitSignal(signal: PerceptionSignal): void { + const key = `emit.${signal.type}.${signal.metadata.action || 'unknown'}` this.emittedSinceStats[key] = (this.emittedSinceStats[key] ?? 0) + 1 - if (payload.kind === 'player') { - this.deps.logger.withFields({ - kind: payload.kind, - action: payload.playerAction, - playerName: payload.playerName, - distance: payload.distance, - hasLineOfSight: payload.hasLineOfSight, - }).log('AttentionDetector: emit') - } - else { - this.deps.logger.withFields({ - kind: payload.kind, - mobName: payload.mobName, - action: payload.mobAction, - distance: payload.distance, - hasLineOfSight: payload.hasLineOfSight, - }).log('AttentionDetector: emit') - } + this.deps.logger.withFields({ + type: signal.type, + desc: signal.description, + meta: signal.metadata, + }).log('AttentionDetector: emit') - this.deps.onAttention(payload) + this.deps.onAttention(signal) } private getBucket(key: string, config: { capacity: number, leakPerSecond: number, trigger: number }): LeakyBucket { diff --git a/services/minecraft/src/cognitive/perception/normalizer-stage.ts b/services/minecraft/src/cognitive/perception/normalizer-stage.ts deleted file mode 100644 index 22c39161c..000000000 --- a/services/minecraft/src/cognitive/perception/normalizer-stage.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { PerceptionFrame } from './frame' -import type { RawPerceptionEvent } from './types/raw-events' -import type { PerceptionStage } from './types/stage' - -function getDistance(raw: RawPerceptionEvent): number | undefined { - return (raw as any).distance -} - -function getEntityId(raw: RawPerceptionEvent): string | undefined { - return (raw as any).entityId -} - -function getDisplayName(raw: RawPerceptionEvent): string | undefined { - return (raw as any).displayName -} - -export class NormalizerStage implements PerceptionStage { - public readonly name = 'normalizer' - - private readonly lastMovedEmitAt = new Map() - private readonly lastSneakValue = new Map() - - constructor( - private readonly deps: { - maxDistance: number - }, - ) { } - - public handle(frame: PerceptionFrame): PerceptionFrame | null { - if (frame.kind !== 'world_raw') - return frame - - const raw = frame.raw as RawPerceptionEvent - - const distance = getDistance(raw) - if (typeof distance === 'number') { - if (distance > this.deps.maxDistance) - return null - - frame.norm = { - ...frame.norm, - distance, - within32: distance <= this.deps.maxDistance, - entityId: getEntityId(raw), - displayName: getDisplayName(raw), - } - - // Drop expensive LOS computation for now. - if (raw.modality === 'sighted') { - ; (raw as any).hasLineOfSight = distance <= this.deps.maxDistance - } - } - - // Throttle spammy entity movement signals - if (raw.modality === 'sighted' && (raw as any).kind === 'entity_moved') { - const entityId = getEntityId(raw) ?? 'unknown' - const now = Date.now() - const last = this.lastMovedEmitAt.get(entityId) - if (typeof last === 'number' && now - last < 100) - return null - this.lastMovedEmitAt.set(entityId, now) - } - - // Dedupe sneak toggle state (mineflayer may emit frequent updates) - if (raw.modality === 'sighted' && (raw as any).kind === 'sneak_toggle') { - const entityId = getEntityId(raw) ?? 'unknown' - const sneaking = !!(raw as any).sneaking - const prev = this.lastSneakValue.get(entityId) - if (prev === sneaking) - return null - this.lastSneakValue.set(entityId, sneaking) - } - - return frame - } -} diff --git a/services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts b/services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts index b39bbbec2..814dfaca8 100644 --- a/services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts +++ b/services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts @@ -78,16 +78,20 @@ describe('perceptionPipeline (e2e)', () => { .filter(e => e.type === 'perception') expect(perceptionEvents.length).toBeGreaterThanOrEqual(1) - expect((perceptionEvents[0] as any).payload).toMatchObject({ + + // Updated assertion for PerceptionSignal + const signal = (perceptionEvents[0] as any).payload + expect(signal.type).toBe('entity_attention') + expect(signal.metadata).toMatchObject({ kind: 'player', - playerAction: 'punch', - playerName: 'alice', + action: 'punch', + displayName: 'alice', }) pipeline.destroy() }) - it('router emits stimulus for chat frames ingested into pipeline', () => { + it('router emits perception signal for chat frames ingested into pipeline', () => { const logger = makeLogger() const eventManager = new EventManager() const emitSpy = vi.spyOn(eventManager, 'emit') @@ -100,14 +104,20 @@ describe('perceptionPipeline (e2e)', () => { pipeline.ingest(createPerceptionFrameFromChat('alice', 'hi')) pipeline.tick(0) - const stimulusEvents = emitSpy.mock.calls + // Should now be a 'perception' event + const perceptionEvents = emitSpy.mock.calls .map(c => c[0]) - .filter(e => e.type === 'stimulus') + .filter(e => e.type === 'perception') - expect(stimulusEvents.length).toBe(1) - expect((stimulusEvents[0] as any).payload).toMatchObject({ - content: 'hi', - metadata: { displayName: 'alice' }, + expect(perceptionEvents.length).toBe(1) + + const signal = (perceptionEvents[0] as any).payload + expect(signal.type).toBe('chat_message') + expect(signal.description).toContain('alice') + expect(signal.description).toContain('hi') + expect(signal.metadata).toMatchObject({ + username: 'alice', + message: 'hi', }) pipeline.destroy() diff --git a/services/minecraft/src/cognitive/perception/pipeline.ts b/services/minecraft/src/cognitive/perception/pipeline.ts index 999008aa2..3f4df236c 100644 --- a/services/minecraft/src/cognitive/perception/pipeline.ts +++ b/services/minecraft/src/cognitive/perception/pipeline.ts @@ -1,16 +1,14 @@ import type { Logg } from '@guiiai/logg' import type { MineflayerWithAgents } from '../types' -import type { StimulusPayload } from '../types' import type { EventManager } from './event-manager' import type { PerceptionFrame } from './frame' -import type { AttentionEventPayload } from './attention-detector' +import type { PerceptionSignal } from './types/signals' import { AttentionDetector } from './attention-detector' import { createPerceptionFrameFromRawEvent } from './frame' import { MineflayerPerceptionCollector } from './mineflayer-perception-collector' -import { NormalizerStage } from './normalizer-stage' import { RawEventBuffer } from './raw-event-buffer' import type { PerceptionStage } from './types/stage' @@ -36,20 +34,17 @@ export class PerceptionPipeline { ) { this.detector = new AttentionDetector({ logger: this.deps.logger, - onAttention: (payload) => { + onAttention: (signal) => { // This is only called synchronously while we're handling a specific frame. // Attach derived signals to that frame; router stage will emit them. this.currentFrame?.signals.push({ - type: 'attention', - payload, + type: 'perception_signal', + payload: signal, }) }, }) this.stages = [ - new NormalizerStage({ - maxDistance: 32, - }), { name: 'attention', tick: (deltaMs) => { @@ -75,14 +70,23 @@ export class PerceptionPipeline { handle: (frame) => { if (frame.kind === 'chat_raw') { const raw = frame.raw as { username: string, message: string } - this.deps.eventManager.emit({ - type: 'stimulus', - payload: { - content: raw.message, - metadata: { - displayName: raw.username, - }, + + // Convert chat to PerceptionSignal + const signal: PerceptionSignal = { + type: 'chat_message', + description: `Chat from ${raw.username}: "${raw.message}"`, + sourceId: raw.username, + timestamp: Date.now(), + confidence: 1.0, + metadata: { + username: raw.username, + message: raw.message, }, + } + + this.deps.eventManager.emit({ + type: 'perception', + payload: signal, source: { type: 'minecraft', id: raw.username, @@ -91,16 +95,16 @@ export class PerceptionPipeline { }) } - // Emit all attention signals centrally as BotEvents - for (const signal of frame.signals) { - if (signal.type !== 'attention') + // Emit all perception signals centrally as BotEvents + for (const signalWrapper of frame.signals) { + if (signalWrapper.type !== 'perception_signal') continue - const payload = signal.payload as AttentionEventPayload + const signal = signalWrapper.payload as PerceptionSignal - this.deps.eventManager.emit({ + this.deps.eventManager.emit({ type: 'perception', - payload, + payload: signal, source: { type: 'minecraft', id: 'perception' }, timestamp: Date.now(), }) diff --git a/services/minecraft/src/cognitive/perception/types/signals.ts b/services/minecraft/src/cognitive/perception/types/signals.ts new file mode 100644 index 000000000..4ae17b8fa --- /dev/null +++ b/services/minecraft/src/cognitive/perception/types/signals.ts @@ -0,0 +1,19 @@ + +export type PerceptionSignalType = + | 'chat_message' + | 'entity_attention' // e.g. someone waving, teabagging + | 'environmental_anomaly' // e.g. sudden loud sound + | 'saliency_high' // generic high saliency event + +export interface PerceptionSignal { + type: PerceptionSignalType + description: string // Textual summary for LLM + + // Contextual Data + sourceId?: string // Who/What caused this + confidence?: number // 0-1 + timestamp: number + + // Structured Data (for logic) + metadata: Record +} diff --git a/services/minecraft/src/cognitive/reflex/reflex-manager.ts b/services/minecraft/src/cognitive/reflex/reflex-manager.ts index c14bc3b47..8993b2f50 100644 --- a/services/minecraft/src/cognitive/reflex/reflex-manager.ts +++ b/services/minecraft/src/cognitive/reflex/reflex-manager.ts @@ -1,7 +1,8 @@ import type { Logg } from '@guiiai/logg' import type { EventManager } from '../perception/event-manager' -import type { BotEvent, MineflayerWithAgents, StimulusPayload } from '../types' +import type { PerceptionSignal } from '../perception/types/signals' +import type { BotEvent, MineflayerWithAgents } from '../types' import type { ReflexContextState } from './context' @@ -12,8 +13,8 @@ export class ReflexManager { private bot: MineflayerWithAgents | null = null private readonly runtime: ReflexRuntime - private readonly onStimulusHandler = (event: BotEvent) => { - this.onStimulus(event) + private readonly onPerceptionHandler = (event: BotEvent) => { + this.onPerception(event) } constructor( @@ -31,11 +32,11 @@ export class ReflexManager { public init(bot: MineflayerWithAgents): void { this.bot = bot - this.deps.eventManager.on('stimulus', this.onStimulusHandler) + this.deps.eventManager.on('perception', this.onPerceptionHandler) } public destroy(): void { - this.deps.eventManager.off('stimulus', this.onStimulusHandler) + this.deps.eventManager.off('perception', this.onPerceptionHandler) this.bot = null } @@ -50,17 +51,23 @@ export class ReflexManager { return this.runtime.getContext().getSnapshot() } - private onStimulus(event: BotEvent): void { + private onPerception(event: BotEvent): void { const bot = this.bot if (!bot) return + const signal = event.payload + // Only care about chat messages for now for social context + if (signal.type !== 'chat_message') + return + const now = Date.now() + const message = signal.metadata.message || signal.description this.runtime.getContext().updateNow(now) this.runtime.getContext().updateSocial({ lastSpeaker: event.source.id, - lastMessage: event.payload.content, + lastMessage: message, lastMessageAt: now, }) diff --git a/services/minecraft/src/cognitive/types/index.ts b/services/minecraft/src/cognitive/types/index.ts index 9af6a9919..2ee117abe 100644 --- a/services/minecraft/src/cognitive/types/index.ts +++ b/services/minecraft/src/cognitive/types/index.ts @@ -29,7 +29,7 @@ export interface CognitiveEngineOptions { } // TODO: currently stimulus is just chat events, consider renaming to 'input' or 'user_interaction' -export type EventCategory = 'stimulus' | 'perception' | 'feedback' | 'world_update' | 'system_alert' +export type EventCategory = 'perception' | 'feedback' | 'world_update' | 'system_alert' export interface BotEventSource { type: 'minecraft' | 'airi' | 'system' @@ -47,14 +47,6 @@ export interface BotEvent { handled?: boolean // Set by Reflex layer to inhibit Conscious layer } -export interface StimulusPayload { - content: string - metadata?: { - entity?: any // prismarine-entity Entity - displayName?: string - } -} - export interface WorldUpdatePayload { event: string data: any