diff --git a/services/minecraft/src/cognitive/perception/events/definitions/arm-swing.ts b/services/minecraft/src/cognitive/perception/events/definitions/arm-swing.ts index 1c30a6550..a46d76bdd 100644 --- a/services/minecraft/src/cognitive/perception/events/definitions/arm-swing.ts +++ b/services/minecraft/src/cognitive/perception/events/definitions/arm-swing.ts @@ -34,22 +34,4 @@ export const armSwingEvent = definePerceptionEvent<[any], ArmSwingExtract>({ }), }, - saliency: { - threshold: 5, - key: 'punch:player', - }, - - signal: { - type: 'entity_attention', - description: extracted => `Player ${extracted.displayName || 'unknown'} is punching nearby`, - metadata: extracted => ({ - kind: 'player', - action: 'punch', - distance: extracted.distance, - hasLineOfSight: extracted.hasLineOfSight, - displayName: extracted.displayName, - }), - }, - - routes: ['conscious', 'reflex', 'debug'], }) diff --git a/services/minecraft/src/cognitive/perception/events/definitions/damage-taken.ts b/services/minecraft/src/cognitive/perception/events/definitions/damage-taken.ts index e842b1c86..b966b667d 100644 --- a/services/minecraft/src/cognitive/perception/events/definitions/damage-taken.ts +++ b/services/minecraft/src/cognitive/perception/events/definitions/damage-taken.ts @@ -86,23 +86,6 @@ function inferCauseFromName(name: string): DamageSourceCause { return 'unknown' } -function describeDamageTaken(extracted: DamageTakenExtract): string { - const amount = extracted.amount - const ds = extracted.damageSource - const cause = ds?.cause ?? 'unknown' - const name = ds?.name - const distance = ds?.distance - - const details = [ - `amount=${amount}`, - `cause=${cause}`, - typeof name === 'string' && name.length > 0 ? `name=${name}` : null, - typeof distance === 'number' ? `distance=${distance.toFixed(1)}` : null, - ].filter(Boolean).join(', ') - - return `Taken damage (${details}).` -} - export const damageTakenEvent = definePerceptionEvent<[], DamageTakenExtract>({ id: 'damage_taken', modality: 'felt', @@ -139,21 +122,4 @@ export const damageTakenEvent = definePerceptionEvent<[], DamageTakenExtract>({ }, }, - saliency: { - threshold: 1, - key: 'felt:damage', - }, - - signal: { - type: 'saliency_high', - description: extracted => describeDamageTaken(extracted), - metadata: extracted => ({ - kind: 'felt', - action: 'damage', - amount: extracted.amount, - damageSource: extracted.damageSource, - }), - }, - - routes: ['conscious', 'reflex', 'debug'], }) diff --git a/services/minecraft/src/cognitive/perception/events/definitions/entity-moved.ts b/services/minecraft/src/cognitive/perception/events/definitions/entity-moved.ts index 24e8cbcfc..48b0602f0 100644 --- a/services/minecraft/src/cognitive/perception/events/definitions/entity-moved.ts +++ b/services/minecraft/src/cognitive/perception/events/definitions/entity-moved.ts @@ -34,22 +34,4 @@ export const entityMovedEvent = definePerceptionEvent<[any], EntityMovedExtract> }), }, - saliency: { - threshold: 5, - key: 'move:player', - }, - - signal: { - type: 'entity_attention', - description: extracted => `Player ${extracted.displayName || 'unknown'} is moving nearby`, - metadata: extracted => ({ - kind: 'player', - action: 'move', - distance: extracted.distance, - hasLineOfSight: extracted.hasLineOfSight, - displayName: extracted.displayName, - }), - }, - - routes: ['reflex', 'debug'], }) diff --git a/services/minecraft/src/cognitive/perception/events/definitions/item-collected.ts b/services/minecraft/src/cognitive/perception/events/definitions/item-collected.ts index 5441a183d..2ada07638 100644 --- a/services/minecraft/src/cognitive/perception/events/definitions/item-collected.ts +++ b/services/minecraft/src/cognitive/perception/events/definitions/item-collected.ts @@ -21,56 +21,4 @@ export const itemCollectedEvent = definePerceptionEvent<[any, any], ItemCollecte }), }, - saliency: { - threshold: 3, - key: 'felt:pickup', - }, - - signal: { - type: 'entity_attention', - description: () => 'Picked up an item', - metadata: extracted => ({ - kind: 'felt', - action: 'pickup', - itemName: extracted.itemName, - }), - }, - - routes: ['conscious', 'debug'], }) - -// TODO: find out if this is needed. Some servers might use entityCollect instead of playerCollect? -// export const itemCollectedEntityCollectEvent = definePerceptionEvent<[any, any], ItemCollectedExtract>({ -// id: 'item_collected_entity_collect', -// modality: 'felt', -// kind: 'item_collected', - -// mineflayer: { -// event: 'entityCollect', -// filter: (ctx, collector, _collected) => { -// if (!collector) -// return false -// return collector.username === ctx.selfUsername -// }, -// extract: (_ctx, _collector, collected) => ({ -// itemName: String(collected?.name ?? collected?.displayName ?? collected?.type ?? 'unknown'), -// }), -// }, - -// saliency: { -// threshold: 3, -// key: 'felt:pickup', -// }, - -// signal: { -// type: 'entity_attention', -// description: () => 'Picked up an item', -// metadata: extracted => ({ -// kind: 'felt', -// action: 'pickup', -// itemName: extracted.itemName, -// }), -// }, - -// routes: ['conscious', 'debug'], -// }) diff --git a/services/minecraft/src/cognitive/perception/events/definitions/sneak-toggle.ts b/services/minecraft/src/cognitive/perception/events/definitions/sneak-toggle.ts index 3452a8989..e980298a3 100644 --- a/services/minecraft/src/cognitive/perception/events/definitions/sneak-toggle.ts +++ b/services/minecraft/src/cognitive/perception/events/definitions/sneak-toggle.ts @@ -59,22 +59,4 @@ export const sneakToggleEvent = definePerceptionEvent<[any], SneakToggleExtract> }), }, - saliency: { - threshold: 5, - key: 'teabag:player', - }, - - signal: { - type: 'entity_attention', - description: extracted => `Player ${extracted.displayName || 'unknown'} is teabagging (rapid sneaking)`, - metadata: extracted => ({ - kind: 'player', - action: 'teabag', - distance: extracted.distance, - hasLineOfSight: extracted.hasLineOfSight, - displayName: extracted.displayName, - }), - }, - - routes: ['conscious', 'reflex', 'debug'], }) diff --git a/services/minecraft/src/cognitive/perception/events/definitions/sound-heard.ts b/services/minecraft/src/cognitive/perception/events/definitions/sound-heard.ts index 1849ce246..eb546bee9 100644 --- a/services/minecraft/src/cognitive/perception/events/definitions/sound-heard.ts +++ b/services/minecraft/src/cognitive/perception/events/definitions/sound-heard.ts @@ -28,21 +28,4 @@ export const soundHeardEvent = definePerceptionEvent<[string, Vec3], SoundHeardE }), }, - saliency: { - threshold: 5, - key: 'sound:ambient', - }, - - signal: { - type: 'environmental_anomaly', - description: extracted => `Heard sound: ${extracted.soundId}`, - metadata: extracted => ({ - kind: 'sound', - action: 'sound', - soundId: extracted.soundId, - distance: extracted.distance, - }), - }, - - routes: ['conscious', 'debug'], }) diff --git a/services/minecraft/src/cognitive/perception/events/definitions/system-message.ts b/services/minecraft/src/cognitive/perception/events/definitions/system-message.ts index c8c0780bc..57ab3b0a7 100644 --- a/services/minecraft/src/cognitive/perception/events/definitions/system-message.ts +++ b/services/minecraft/src/cognitive/perception/events/definitions/system-message.ts @@ -11,19 +11,4 @@ export const systemMessageEvent = definePerceptionEvent<[string, string], { mess extract: (_ctx, message, position) => ({ message, position }), }, - saliency: { - threshold: 1, - key: 'system:message', - }, - - signal: { - type: 'system_message', - description: extracted => extracted.message, - metadata: extracted => ({ - message: extracted.message, - position: extracted.position, - }), - }, - - routes: ['reflex'], }) diff --git a/services/minecraft/src/cognitive/perception/events/index.ts b/services/minecraft/src/cognitive/perception/events/index.ts index 8b477c9b5..c0262f07c 100644 --- a/services/minecraft/src/cognitive/perception/events/index.ts +++ b/services/minecraft/src/cognitive/perception/events/index.ts @@ -5,7 +5,6 @@ import type { Vec3 } from 'vec3' import type { PerceptionContext, PerceptionEventDefinition, - PerceptionSignal, RawPerceptionEventBase, } from './types' @@ -15,17 +14,6 @@ export function definePerceptionEvent( return definition } -interface WindowCounter { - head: number - windowSize: number - counts: number[] - triggers: number[] - total: number - lastEventSlot: number - lastFireSlot: number | null - lastFireTotal: number -} - interface RegisteredListener { event: string handler: (...args: any[]) => void @@ -33,33 +21,12 @@ interface RegisteredListener { export interface EventRegistryDeps { logger: Logg - onSignal: (signal: PerceptionSignal) => void - onRawEvent?: (event: RawPerceptionEventBase & Record) => void -} - -const SLOT_MS = 20 -const DEFAULT_WINDOW_MS = 2000 - -export interface CounterSnapshot { - key: string - total: number - window: number[] - triggers: number[] - lastFireSlot: number | null - lastFireTotal: number -} - -export interface SaliencySnapshot { - slot: number - counters: CounterSnapshot[] + onRawEvent: (event: RawPerceptionEventBase & Record) => void } export class EventRegistry { private definitions: Map = new Map() - private counters: Map = new Map() private listeners: RegisteredListener[] = [] - private currentSlot = 0 - private timer: ReturnType | null = null private context: PerceptionContext | null = null private maxDistance = 32 @@ -67,12 +34,6 @@ export class EventRegistry { public register(definition: PerceptionEventDefinition): void { this.definitions.set(definition.id, definition) - - if (definition.saliency) { - const windowMs = definition.saliency.windowMs ?? DEFAULT_WINDOW_MS - const windowSize = Math.max(1, Math.round(windowMs / SLOT_MS)) - this.counters.set(definition.saliency.key, this.createCounter(windowSize)) - } } public registerAll(definitions: PerceptionEventDefinition[]): void { @@ -81,47 +42,13 @@ export class EventRegistry { } } - public start(): void { - if (this.timer) - return - - this.timer = setInterval(() => { - this.currentSlot += 1 - this.advanceWindows() - }, SLOT_MS) - } - public stop(): void { - if (this.timer) { - clearInterval(this.timer) - this.timer = null - } // NOTICE: Nullify context so that any stale mineflayer listeners that fire // after stop() (but before detachFromBot()) are silently ignored by the // guard at the top of handleMineflayerEvent. this.context = null } - public getDebugSnapshot(): SaliencySnapshot { - const counters: CounterSnapshot[] = [] - - for (const [key, counter] of this.counters.entries()) { - counters.push({ - key, - total: counter.total, - window: this.exportWindow(counter), - triggers: this.exportTriggers(counter), - lastFireSlot: counter.lastFireSlot, - lastFireTotal: counter.lastFireTotal, - }) - } - - return { - slot: this.currentSlot, - counters, - } - } - public attachToBot(bot: Bot, maxDistance = 32): void { this.maxDistance = maxDistance this.context = this.createContext(bot) @@ -175,22 +102,10 @@ export class EventRegistry { } } - public getSignalTypes(): string[] { - const types = new Set() - for (const def of this.definitions.values()) { - types.add(def.signal.type) - } - return Array.from(types) - } - public getDefinitions(): PerceptionEventDefinition[] { return Array.from(this.definitions.values()) } - public getSaliencyKeys(): string[] { - return Array.from(this.counters.keys()) - } - private handleMineflayerEvent(def: PerceptionEventDefinition, args: any[]): void { if (!this.context) return @@ -210,95 +125,7 @@ export class EventRegistry { ...extracted, } - if (def.routes.includes('reflex')) - this.deps.onRawEvent?.(rawEvent) - - if (def.saliency) { - const shouldEmit = this.incrementAndCheck(def.saliency.key, def.saliency.threshold) - if (!shouldEmit) - return - } - - const signal: PerceptionSignal = { - type: def.signal.type, - description: def.signal.description(extracted, timestamp), - sourceId: def.id, - confidence: 1.0, - timestamp, - metadata: def.signal.metadata?.(extracted) ?? extracted, - } - - if (def.routes.includes('conscious')) - this.deps.onSignal(signal) - } - - private incrementAndCheck(key: string, threshold: number): boolean { - const counter = this.counters.get(key) - if (!counter) - return true - - counter.counts[counter.head] = (counter.counts[counter.head] ?? 0) + 1 - counter.total += 1 - counter.lastEventSlot = this.currentSlot - - if (counter.total >= threshold) { - counter.lastFireSlot = this.currentSlot - counter.lastFireTotal = counter.total - counter.triggers[counter.head] = 1 - this.resetCounter(counter) - return true - } - - return false - } - - private createCounter(windowSize: number): WindowCounter { - return { - head: 0, - windowSize, - counts: new Array(windowSize).fill(0), - triggers: new Array(windowSize).fill(0), - total: 0, - lastEventSlot: 0, - lastFireSlot: null, - lastFireTotal: 0, - } - } - - private resetCounter(counter: WindowCounter): void { - counter.total = 0 - counter.counts.fill(0) - // do not reset triggers; they are historical markers for debug - } - - private advanceWindows(): void { - for (const counter of this.counters.values()) { - counter.head = (counter.head + 1) % counter.windowSize - const expired = counter.counts[counter.head] ?? 0 - if (expired > 0) { - counter.total = Math.max(0, counter.total - expired) - } - counter.counts[counter.head] = 0 - counter.triggers[counter.head] = 0 - } - } - - private exportWindow(counter: WindowCounter): number[] { - const out = new Array(counter.windowSize) - for (let i = 0; i < counter.windowSize; i++) { - const idx = (counter.head + 1 + i) % counter.windowSize - out[i] = counter.counts[idx] ?? 0 - } - return out - } - - private exportTriggers(counter: WindowCounter): number[] { - const out = new Array(counter.windowSize) - for (let i = 0; i < counter.windowSize; i++) { - const idx = (counter.head + 1 + i) % counter.windowSize - out[i] = counter.triggers[idx] ?? 0 - } - return out + this.deps.onRawEvent(rawEvent) } } diff --git a/services/minecraft/src/cognitive/perception/events/types.ts b/services/minecraft/src/cognitive/perception/events/types.ts index da668b772..71ac0344f 100644 --- a/services/minecraft/src/cognitive/perception/events/types.ts +++ b/services/minecraft/src/cognitive/perception/events/types.ts @@ -3,8 +3,6 @@ import type { Vec3 } from 'vec3' export type PerceptionModality = 'sighted' | 'heard' | 'felt' | 'system' -export type PerceptionRoute = 'reflex' | 'conscious' | 'debug' - export interface PerceptionContext { bot: Bot selfUsername: string @@ -21,30 +19,12 @@ export interface MineflayerBinding extract: (ctx: PerceptionContext, ...args: TArgs) => TExtract } -export interface SaliencyConfig { - threshold: number - key: string - windowMs?: number -} - -export interface SignalConfig { - type: string - description: (extracted: TExtract, timestamp: number) => string - metadata?: (extracted: TExtract) => Record -} - export interface PerceptionEventDefinition { id: string modality: PerceptionModality kind: string mineflayer: MineflayerBinding - - saliency?: SaliencyConfig - - signal: SignalConfig - - routes: PerceptionRoute[] } export interface RawPerceptionEventBase { @@ -53,12 +33,3 @@ export interface RawPerceptionEventBase { timestamp: number source: string } - -export interface PerceptionSignal { - type: string - description: string - sourceId?: string - confidence: number - timestamp: number - metadata: Record -} diff --git a/services/minecraft/src/cognitive/perception/frame.ts b/services/minecraft/src/cognitive/perception/frame.ts deleted file mode 100644 index 9a668a887..000000000 --- a/services/minecraft/src/cognitive/perception/frame.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { RawPerceptionEvent } from './types/raw-events' - -export type PerceptionFrameSource = 'minecraft' - -export type PerceptionFrameKind - = | 'world_raw' - | 'chat_raw' - -export interface PerceptionFrame { - id: string - ts: number - source: PerceptionFrameSource - kind: PerceptionFrameKind - raw: RawPerceptionEvent | { username: string, message: string } - signals: Array<{ type: string, payload: any }> -} - -export function createPerceptionFrameFromRawEvent(event: RawPerceptionEvent): PerceptionFrame { - return { - id: `p_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`, - ts: event.timestamp, - source: 'minecraft', - kind: 'world_raw', - raw: event, - signals: [], - } -} - -export function createPerceptionFrameFromChat(username: string, message: string): PerceptionFrame { - return { - id: `p_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`, - ts: Date.now(), - source: 'minecraft', - kind: 'chat_raw', - raw: { - username, - message, - }, - signals: [], - } -} diff --git a/services/minecraft/src/cognitive/perception/leaky-bucket.ts b/services/minecraft/src/cognitive/perception/leaky-bucket.ts deleted file mode 100644 index af8dfe619..000000000 --- a/services/minecraft/src/cognitive/perception/leaky-bucket.ts +++ /dev/null @@ -1,35 +0,0 @@ -export interface LeakyBucketConfig { - capacity: number - leakPerSecond: number - trigger: number -} - -export class LeakyBucket { // Shoutout to Makito for this idea - private value = 0 - private readonly capacity: number - private readonly leakPerMs: number - private readonly trigger: number - - constructor(config: LeakyBucketConfig) { - this.capacity = config.capacity - this.leakPerMs = config.leakPerSecond / 1000 - this.trigger = config.trigger - } - - public tick(deltaMs: number): void { - if (deltaMs <= 0) - return - this.value = Math.max(0, this.value - this.leakPerMs * deltaMs) - } - - public add(amount: number): { fired: boolean, value: number } { - const next = Math.min(this.capacity, this.value + amount) - const fired = this.value < this.trigger && next >= this.trigger - this.value = next - return { fired, value: this.value } - } - - public getValue(): number { - return this.value - } -} diff --git a/services/minecraft/src/cognitive/perception/mineflayer-perception-collector.ts b/services/minecraft/src/cognitive/perception/mineflayer-perception-collector.ts deleted file mode 100644 index b3ad921bb..000000000 --- a/services/minecraft/src/cognitive/perception/mineflayer-perception-collector.ts +++ /dev/null @@ -1,442 +0,0 @@ -import type { Logg } from '@guiiai/logg' -import type { Vec3 } from 'vec3' - -import type { MineflayerWithAgents } from '../types' -import type { - FeltDamageTakenEvent, - FeltItemCollectedEvent, - HeardSoundEvent, - PlayerJoinedEvent, - RawPerceptionEvent, - SightedArmSwingEvent, - SightedEntityMovedEvent, - SightedSneakToggleEvent, - SystemMessageEvent, -} from './types/raw-events' - -export class MineflayerPerceptionCollector { - private bot: MineflayerWithAgents | null = null - private readonly listeners: Array<{ - event: string - handler: (...args: any[]) => void - }> = [] - - private knownPlayerIds: Set = new Set() - - private lastSelfHealth: number | null = null - private lastStatsAt = 0 - private stats: Record = {} - private sneakingState: Map = new Map() - - constructor( - private readonly deps: { - logger: Logg - emitRaw: (event: RawPerceptionEvent) => void - maxDistance: number - }, - ) { } - - public init(bot: MineflayerWithAgents): void { - this.bot = bot - this.lastSelfHealth = bot.bot.health - this.lastStatsAt = Date.now() - this.stats = {} - this.knownPlayerIds = this.snapshotKnownPlayers(bot) - - this.deps.logger.withFields({ maxDistance: this.deps.maxDistance }).log('MineflayerPerceptionCollector: init') - - this.registerEventHandlers() - } - - public destroy(): void { - if (!this.bot) - return - - this.deps.logger.withFields({ listeners: this.listeners.length }).log('MineflayerPerceptionCollector: destroy') - - for (const { event, handler } of this.listeners) { - try { - const b = this.bot.bot as any - b.off?.(event, handler) - b.removeListener?.(event, handler) - } - catch (err) { - this.deps.logger.withError(err as Error).error('MineflayerPerceptionCollector: failed to remove listener') - } - } - - this.listeners.length = 0 - this.lastSelfHealth = null - this.bot = null - } - - // ======================================== - // Event Handler Registration - // ======================================== - - private registerEventHandlers(): void { - // NOTICE: Sighted events (entityMoved, entitySwingArm, entityUpdate) now handled by EventRegistry - // See perception/events/definitions/arm-swing.ts, sneak-toggle.ts, entity-moved.ts - // NOTICE: Heard/Felt events (soundEffectHeard, health, playerCollect) now handled by EventRegistry - // See perception/events/definitions/sound-heard.ts, damage-taken.ts, item-collected.ts - // NOTICE: playerJoined hook disconnected - system messages now cover join/leave events - // this.onBot('playerJoined', player => this.handlePlayerJoined(player)) - // this.onBot('playerUpdated', () => this.handlePlayersMaybeChanged()) - // NOTICE: messagestr now handled by EventRegistry in perception/events/definitions/system-message.ts - } - - // ======================================== - // Sighted Event Handlers - // NOTICE: These handlers are now handled by EventRegistry - // Kept for reference during migration - // ======================================== - - // @ts-expect-error Intentionally unused - migrated to EventRegistry - private handleEntityMoved(entity: any): void { - if (!this.isValidEntityInRange(entity)) - return - - const event: SightedEntityMovedEvent = { - modality: 'sighted', - kind: 'entity_moved', - entityType: entity?.type === 'player' ? 'player' : 'mob', - entityId: this.entityId(entity), - displayName: entity?.username, - distance: this.distanceTo(entity)!, - hasLineOfSight: true, - timestamp: Date.now(), - source: 'minecraft', - pos: entity?.position, - } - - this.emitEvent(event, 'sighted.entity_moved') - } - - // @ts-expect-error Intentionally unused - migrated to EventRegistry - private handleEntitySwingArm(entity: any): void { - if (!this.isValidEntityInRange(entity)) - return - - const event: SightedArmSwingEvent = { - modality: 'sighted', - kind: 'arm_swing', - entityType: 'player', - entityId: this.entityId(entity), - displayName: entity?.username, - distance: this.distanceTo(entity)!, - hasLineOfSight: true, - timestamp: Date.now(), - source: 'minecraft', - pos: entity?.position, - } - - this.emitEvent(event, 'sighted.arm_swing') - } - - // @ts-expect-error Intentionally unused - migrated to EventRegistry - private handleEntityUpdate(entity: any): void { - if (!entity || entity.type !== 'player') - return - - if (this.isSelfEntity(entity)) - return - - const entityId = this.entityId(entity) - const isSneaking = this.extractSneakingState(entity) - - if (!this.hasSneakingStateChanged(entityId, isSneaking)) - return - - this.sneakingState.set(entityId, isSneaking) - - const dist = this.distanceTo(entity) - if (dist === null || dist > this.deps.maxDistance) - return - - const event: SightedSneakToggleEvent = { - modality: 'sighted', - kind: 'sneak_toggle', - entityType: 'player', - entityId, - displayName: entity?.username, - distance: dist, - hasLineOfSight: true, - sneaking: isSneaking, - timestamp: Date.now(), - source: 'minecraft', - pos: entity?.position, - } - - this.emitEvent(event, 'sighted.sneak_toggle') - } - - // ======================================== - // Heard Event Handlers - // NOTICE: These handlers are now handled by EventRegistry - // ======================================== - - // @ts-expect-error Intentionally unused - migrated to EventRegistry - private handleSoundHeard(soundId: string, pos: Vec3): void { - if (!pos) - return - - const dist = this.distanceToPos(pos) - if (dist === null || dist > this.deps.maxDistance) - return - - const event: HeardSoundEvent = { - modality: 'heard', - kind: 'sound', - soundId, - distance: dist, - timestamp: Date.now(), - source: 'minecraft', - pos, - } - - this.deps.emitRaw(event) - } - - // ======================================== - // Felt Event Handlers - // NOTICE: These handlers are now handled by EventRegistry - // ======================================== - - // @ts-expect-error Intentionally unused - migrated to EventRegistry - private handleHealthChange(): void { - if (!this.bot) - return - - const current = this.bot.bot.health - const prev = this.lastSelfHealth - this.lastSelfHealth = current - - if (typeof prev !== 'number' || current >= prev) - return - - const event: FeltDamageTakenEvent = { - modality: 'felt', - kind: 'damage_taken', - amount: prev - current, - timestamp: Date.now(), - source: 'minecraft', - } - - this.emitEvent(event, 'felt.damage_taken') - } - - // @ts-expect-error Intentionally unused - migrated to EventRegistry - private handleItemCollected(collector: any, collected: any): void { - if (!this.bot || !collector) - return - - if (collector.username !== this.bot.bot.username) - return - - const itemName = String(collected?.name ?? collected?.displayName ?? collected?.type ?? 'unknown') - - const event: FeltItemCollectedEvent = { - modality: 'felt', - kind: 'item_collected', - itemName, - timestamp: Date.now(), - source: 'minecraft', - } - - this.emitEvent(event, 'felt.item_collected') - } - - // ======================================== - // System Event Handlers - // ======================================== - - // NOTICE: handleSystemMessage now handled by EventRegistry - // @ts-expect-error Intentionally unused - kept for reference during migration - private handleSystemMessage(message: string, messagePosition: string): void { - // messagePosition: 'chat' = player chat, 'system' = system message, 'game_info' = action bar - if (messagePosition !== 'system') - return - - const event: SystemMessageEvent = { - modality: 'system', - kind: 'system_message', - message, - position: messagePosition, - timestamp: Date.now(), - source: 'minecraft', - } - - this.emitEvent(event, 'system.system_message') - } - - // NOTICE: handlePlayerJoined and handlePlayersMaybeChanged are disconnected - // System messages now cover join/leave events via messagestr - // @ts-expect-error Intentionally unused - kept for potential future reconnection - private handlePlayerJoined(player: any): void { - if (!player) - return - - if (player.username === this.bot?.bot.username) - return - - const playerId = String(player.uuid ?? player.id ?? player.username ?? 'unknown') - if (this.knownPlayerIds.has(playerId)) - return - - this.knownPlayerIds.add(playerId) - - const event: PlayerJoinedEvent = { - modality: 'system', - kind: 'player_joined', - playerId, - displayName: player.username, - timestamp: Date.now(), - source: 'minecraft', - } - - this.emitEvent(event, 'system.player_joined') - } - - // @ts-expect-error Intentionally unused - kept for potential future reconnection - private handlePlayersMaybeChanged(): void { - const bot = this.bot - if (!bot) - return - - const current = this.snapshotKnownPlayers(bot) - - for (const playerId of current) { - if (this.knownPlayerIds.has(playerId)) - continue - - this.knownPlayerIds.add(playerId) - - const player = bot.bot.players?.[playerId] - const username = player?.username - - const event: PlayerJoinedEvent = { - modality: 'system', - kind: 'player_joined', - playerId, - displayName: typeof username === 'string' ? username : undefined, - timestamp: Date.now(), - source: 'minecraft', - } - - this.emitEvent(event, 'system.player_joined') - } - } - - private snapshotKnownPlayers(bot: MineflayerWithAgents): Set { - const out = new Set() - const players = bot.bot.players as Record | undefined - if (!players) - return out - - const selfUsername = bot.bot.username - - for (const [id, player] of Object.entries(players)) { - if (!id) - continue - - const username = player?.username - if (username && username === selfUsername) - continue - - out.add(String(id)) - } - - return out - } - - // ======================================== - // Validation Helpers - // ======================================== - - private isValidEntityInRange(entity: any): boolean { - const dist = this.distanceTo(entity) - if (dist === null || dist > this.deps.maxDistance) - return false - - if (this.isSelfEntity(entity)) - return false - - return true - } - - private isSelfEntity(entity: any): boolean { - return entity.username === this.bot?.bot.username - } - - private extractSneakingState(entity: any): boolean { - const flags = entity?.metadata?.[0] - // Bit 1 (0x02) is sneaking - return typeof flags === 'number' ? !!(flags & 0x02) : false - } - - private hasSneakingStateChanged(entityId: string, isSneaking: boolean): boolean { - const lastState = this.sneakingState.get(entityId) - return lastState !== isSneaking - } - - // ======================================== - // Utilities - // ======================================== - - private emitEvent(event: RawPerceptionEvent, statKey: string): void { - this.deps.emitRaw(event) - this.bumpStat(statKey) - this.maybeLogStats() - } - - private bumpStat(key: string): void { - this.stats[key] = (this.stats[key] ?? 0) + 1 - } - - private maybeLogStats(): void { - const now = Date.now() - if (now - this.lastStatsAt < 2000) - return - - // this.deps.logger.withFields({ - // ...this.stats, - // }).log('MineflayerPerceptionCollector: stats') - - this.lastStatsAt = now - this.stats = {} - } - - // @ts-expect-error Intentionally unused - all events migrated to EventRegistry - private onBot(event: string, handler: (...args: any[]) => void): void { - if (!this.bot) - return - - (this.bot.bot as any).on(event, handler) - this.listeners.push({ event, handler }) - } - - private entityId(entity: any): string { - return String(entity?.id ?? entity?.uuid ?? entity?.username ?? 'unknown') - } - - private distanceTo(entity: any): number | null { - const pos = entity?.position - if (!pos) - return null - return this.distanceToPos(pos) - } - - private distanceToPos(pos: Vec3): number | null { - if (!this.bot) - return null - const selfPos = this.bot.bot.entity?.position - if (!selfPos) - return null - try { - return selfPos.distanceTo(pos) - } - catch { - return null - } - } -} diff --git a/services/minecraft/src/cognitive/perception/pipeline.ts b/services/minecraft/src/cognitive/perception/pipeline.ts index 861a29d26..d885e90f8 100644 --- a/services/minecraft/src/cognitive/perception/pipeline.ts +++ b/services/minecraft/src/cognitive/perception/pipeline.ts @@ -2,11 +2,7 @@ import type { Logg } from '@guiiai/logg' import type { EventBus } from '../os' import type { MineflayerWithAgents } from '../types' -import type { PerceptionFrame } from './frame' -import type { RawPerceptionEvent } from './types/raw-events' -import type { PerceptionStage } from './types/stage' -import { DebugService } from '../../debug' import { EventRegistry } from './events' import { allEventDefinitions } from './events/definitions' import { PerceptionAPI } from './perception-api' @@ -15,11 +11,6 @@ export class PerceptionPipeline { private readonly perception: PerceptionAPI private readonly eventRegistry: EventRegistry private bot: MineflayerWithAgents | null = null - private initialized = false - - private readonly stages: PerceptionStage[] - - private saliencyEmitTimer: ReturnType | null = null constructor( private readonly deps: { @@ -31,12 +22,6 @@ export class PerceptionPipeline { this.eventRegistry = new EventRegistry({ logger: this.deps.logger, - onSignal: (signal) => { - // NOTICE: In the linear architecture, Perception does not emit signal:* directly. - // Raw events are emitted and the rules layer derives signals. Reflex then forwards - // selected signals to conscious. - void signal - }, onRawEvent: (event) => { const eventType = `raw:${event.modality}:${event.kind}` this.deps.eventBus.emit({ @@ -47,63 +32,14 @@ export class PerceptionPipeline { }, }) this.eventRegistry.registerAll(allEventDefinitions) - - this.stages = [ - { - name: 'entity_update', - handle: (frame) => { - if (frame.kind !== 'world_raw') - return frame - - const raw = frame.raw as RawPerceptionEvent - - // Feed entity updates to PerceptionAPI - if ('entityId' in raw && 'entityType' in raw) { - const entityRaw = raw as RawPerceptionEvent & { entityId: string, entityType: string, displayName?: string, pos?: { x: number, y: number, z: number } } - if (entityRaw.entityType === 'player') { - this.perception.updateEntity(entityRaw.entityId, { - id: entityRaw.entityId, - type: 'player', - name: entityRaw.displayName, - position: entityRaw.pos as any, - isSneaking: 'sneaking' in entityRaw ? (entityRaw as any).sneaking : undefined, - }) - } - } - - return frame - }, - }, - { - name: 'attention', - handle: (frame) => { - if (frame.kind !== 'world_raw') - return frame - - const raw = frame.raw as RawPerceptionEvent - // Legacy pipeline saliency disabled; EventRegistry is the source of truth. - // Keep raw emission for now in case some other component still ingests frames. - this.emitRawToEventBus(raw) - return frame - }, - }, - ] } public init(bot: MineflayerWithAgents): void { - this.initialized = true this.bot = bot this.deps.logger.withFields({ maxDistance: 32 }).log('PerceptionPipeline: init') - this.eventRegistry.start() this.eventRegistry.attachToBot(bot.bot, 32) - - this.saliencyEmitTimer = setInterval(() => { - if (!this.initialized) - return - DebugService.getInstance().emit('saliency', this.eventRegistry.getDebugSnapshot()) - }, 100) } public destroy(): void { @@ -114,12 +50,6 @@ export class PerceptionPipeline { } this.eventRegistry.stop() this.bot = null - - if (this.saliencyEmitTimer) { - clearInterval(this.saliencyEmitTimer) - this.saliencyEmitTimer = null - } - this.initialized = false } /** @@ -128,47 +58,4 @@ export class PerceptionPipeline { public getPerceptionAPI(): PerceptionAPI { return this.perception } - - /** - * Get all registered signal types from the EventRegistry - * Used by consumers (e.g., Brain) to dynamically subscribe to signals - */ - public getSignalTypes(): string[] { - return this.eventRegistry.getSignalTypes() - } - - public ingest(frame: PerceptionFrame): void { - if (!this.initialized) - return - - let current: PerceptionFrame | null = frame - for (const stage of this.stages) { - if (!current) - break - try { - current = stage.handle(current) - } - catch (err) { - this.deps.logger.withError(err as Error).error('PerceptionPipeline: stage error') - break - } - } - } - - /** - * Emit a raw perception event to the EventBus - * This bridges the perception system to the rules layer - */ - private emitRawToEventBus(raw: RawPerceptionEvent): void { - const eventType = `raw:${raw.modality}:${raw.kind}` - - this.deps.eventBus.emit({ - type: eventType, - payload: Object.freeze(raw), - source: { - component: 'perception', - id: raw.source, - }, - }) - } } diff --git a/services/minecraft/src/cognitive/perception/raw-event-buffer.ts b/services/minecraft/src/cognitive/perception/raw-event-buffer.ts deleted file mode 100644 index ea78de06e..000000000 --- a/services/minecraft/src/cognitive/perception/raw-event-buffer.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { PerceptionFrame } from './frame' - -export class RawEventBuffer { - private queue: PerceptionFrame[] = [] - - public push(event: PerceptionFrame): void { - this.queue.push(event) - } - - public size(): number { - return this.queue.length - } - - public drain(): PerceptionFrame[] { - if (this.queue.length === 0) - return [] - const drained = this.queue - this.queue = [] - return drained - } - - public clear(): void { - this.queue = [] - } -} diff --git a/services/minecraft/src/cognitive/perception/raw-events.ts b/services/minecraft/src/cognitive/perception/raw-events.ts deleted file mode 100644 index 8efb34698..000000000 --- a/services/minecraft/src/cognitive/perception/raw-events.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './types/raw-events' diff --git a/services/minecraft/src/cognitive/perception/saliency-detector.ts b/services/minecraft/src/cognitive/perception/saliency-detector.ts deleted file mode 100644 index 50869ae90..000000000 --- a/services/minecraft/src/cognitive/perception/saliency-detector.ts +++ /dev/null @@ -1,243 +0,0 @@ -import type { Logg } from '@guiiai/logg' - -import type { SaliencyRuleBook } from './saliency-rules' -import type { RawPerceptionEvent } from './types/raw-events' -import type { PerceptionSignal } from './types/signals' - -import { EVENT_KEYS, SALIENCY_RULES, WINDOW_SIZE } from './saliency-rules' - -/** - * Circular buffer counter for a single event type - */ -interface WindowCounter { - /** Current head position in circular buffer */ - head: number - /** Event counts per slot */ - counts: number[] - /** Trigger markers per slot (1 = fired in this slot) */ - triggers: number[] - /** Running total of counts in window */ - total: number - /** Slot when last event was received */ - lastEventSlot: number - /** Slot when threshold was last triggered */ - lastFireSlot: number | null - /** Total count when last fired */ - lastFireTotal: number -} - -/** - * Debug snapshot data for a single counter - */ -export interface CounterSnapshot { - key: string - total: number - window: number[] - triggers: number[] - lastFireSlot: number | null - lastFireTotal: number -} - -/** - * Debug snapshot for the entire saliency system - */ -export interface SaliencySnapshot { - slot: number - counters: CounterSnapshot[] -} - -export class SaliencyDetector { - /** Fixed set of counters - one per event key, never deleted */ - private readonly counters: Map = new Map() - - /** Current slot (advances every slotMs) */ - private currentSlot = 0 - - /** Timer for advancing slots */ - private timer: ReturnType | null = null - - /** Milliseconds per slot */ - private readonly slotMs = 20 - - constructor( - private readonly deps: { - logger: Logg - onAttention: (signal: PerceptionSignal) => void - rules?: SaliencyRuleBook - }, - ) { - // Initialize all counters at construction (fixed set) - for (const key of EVENT_KEYS) { - this.counters.set(key, this.createCounter()) - } - } - - public start(): void { - if (this.timer) - return - - this.timer = setInterval(() => { - this.currentSlot += 1 - this.advanceWindows() - }, this.slotMs) - } - - public stop(): void { - if (!this.timer) - return - clearInterval(this.timer) - this.timer = null - } - - /** - * Process an incoming perception event - */ - public ingest(event: RawPerceptionEvent): void { - const rule = this.lookupRule(event) - if (!rule) - return - - if (rule.predicate && !rule.predicate(event)) - return - - const counter = this.counters.get(rule.key) - if (!counter) { - // Unknown key - should not happen with fixed key set - return - } - - // Increment count in current slot - counter.counts[counter.head] = (counter.counts[counter.head] ?? 0) + 1 - counter.total += 1 - counter.lastEventSlot = this.currentSlot - - // Check threshold - if (counter.total >= rule.threshold) { - counter.lastFireSlot = this.currentSlot - counter.lastFireTotal = counter.total - counter.triggers[counter.head] = 1 - this.resetCounter(counter) - this.emitSignal(rule.buildSignal(event)) - } - } - - /** - * Get debug snapshot for visualization - * Returns all counters in fixed order (always same keys, same order) - */ - public getDebugSnapshot(): SaliencySnapshot { - const counters: CounterSnapshot[] = [] - - for (const key of EVENT_KEYS) { - const counter = this.counters.get(key) - if (!counter) - continue - - counters.push({ - key, - total: counter.total, - window: this.exportWindow(counter), - triggers: this.exportTriggers(counter), - lastFireSlot: counter.lastFireSlot, - lastFireTotal: counter.lastFireTotal, - }) - } - - return { - slot: this.currentSlot, - counters, - } - } - - /** - * Advance all windows by one slot - * Never deletes counters - they persist with zeroed values - */ - private advanceWindows(): void { - for (const counter of this.counters.values()) { - // Move head forward in circular buffer - counter.head = (counter.head + 1) % WINDOW_SIZE - - // Subtract expired slot from total - const expired = counter.counts[counter.head] ?? 0 - if (expired > 0) { - counter.total = Math.max(0, counter.total - expired) - } - - // Clear the slot for new data - counter.counts[counter.head] = 0 - counter.triggers[counter.head] = 0 - } - } - - /** - * Create a new counter with zeroed circular buffer - */ - private createCounter(): WindowCounter { - return { - head: 0, - counts: new Array(WINDOW_SIZE).fill(0), - triggers: new Array(WINDOW_SIZE).fill(0), - total: 0, - lastEventSlot: 0, - lastFireSlot: null, - lastFireTotal: 0, - } - } - - /** - * Reset counter counts (after threshold triggered) - * Note: We only reset the counts, not triggers - triggers are visual markers - * that should persist until they naturally expire in the circular buffer - */ - private resetCounter(counter: WindowCounter): void { - counter.total = 0 - counter.counts.fill(0) - // Don't reset triggers - they're historical markers for visualization - } - - /** - * Export window data in chronological order (oldest -> newest) - */ - private exportWindow(counter: WindowCounter): number[] { - const out = new Array(WINDOW_SIZE) - for (let i = 0; i < WINDOW_SIZE; i++) { - const idx = (counter.head + 1 + i) % WINDOW_SIZE - out[i] = counter.counts[idx] ?? 0 - } - return out - } - - /** - * Export trigger markers in chronological order (oldest -> newest) - */ - private exportTriggers(counter: WindowCounter): number[] { - const out = new Array(WINDOW_SIZE) - for (let i = 0; i < WINDOW_SIZE; i++) { - const idx = (counter.head + 1 + i) % WINDOW_SIZE - out[i] = counter.triggers[idx] ?? 0 - } - return out - } - - /** - * Find matching rule for an event - */ - private lookupRule(event: RawPerceptionEvent) { - const rules = this.deps.rules ?? SALIENCY_RULES - return rules[event.modality]?.[event.kind] - } - - /** - * Emit a perception signal via callback - */ - private emitSignal(signal: PerceptionSignal): void { - // this.deps.logger.withFields({ - // type: signal.type, - // desc: signal.description, - // meta: signal.metadata, - // }).log('SaliencyDetector: emit') - - this.deps.onAttention(signal) - } -} diff --git a/services/minecraft/src/cognitive/perception/saliency-rules.ts b/services/minecraft/src/cognitive/perception/saliency-rules.ts deleted file mode 100644 index 28f632970..000000000 --- a/services/minecraft/src/cognitive/perception/saliency-rules.ts +++ /dev/null @@ -1,211 +0,0 @@ -import type { RawPerceptionEvent } from './types/raw-events' -import type { PerceptionSignal } from './types/signals' - -export interface SaliencyRule { - /** - * How many occurrences within the window are required before emitting. - */ - threshold: number - /** - * Optional predicate to gate the rule. - */ - predicate?: (event: E) => boolean - /** - * Counter key - should be one of the predefined EVENT_KEYS - */ - key: string - /** - * Builds the PerceptionSignal when the rule fires. - */ - buildSignal: (event: E) => PerceptionSignal -} - -export type SaliencyRuleBook = Partial ->> - -/** - * Fixed event keys for stable heatmap visualization. - * All counters are initialized with these keys at startup. - */ -export const EVENT_KEYS = [ - 'punch:player', - 'teabag:player', - 'move:player', - 'sound:ambient', - 'felt:damage', - 'felt:pickup', - 'system:message', -] as const - -export type EventKey = typeof EVENT_KEYS[number] - -/** - * Unified window size for all event types (100 slots = ~2 seconds at 20ms/slot) - */ -export const WINDOW_SIZE = 100 - -/** - * Default threshold for attention triggering - */ -export const DEFAULT_THRESHOLD = 5 - -export const SALIENCY_RULES: SaliencyRuleBook = { - sighted: { - arm_swing: { - threshold: 5, - key: 'punch:player', - buildSignal: (event) => { - const e = event as Extract - return { - type: 'entity_attention', - description: `Player ${e.displayName || 'unknown'} is punching nearby`, - sourceId: e.entityId, - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'player', - action: 'punch', - distance: e.distance, - hasLineOfSight: e.hasLineOfSight, - displayName: e.displayName, - }, - } - }, - }, - sneak_toggle: { - threshold: 5, - key: 'teabag:player', - buildSignal: (event) => { - const e = event as Extract - return { - type: 'entity_attention', - description: `Player ${e.displayName || 'unknown'} is teabagging (rapid sneaking)`, - sourceId: e.entityId, - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'player', - action: 'teabag', - distance: e.distance, - hasLineOfSight: e.hasLineOfSight, - displayName: e.displayName, - }, - } - }, - }, - entity_moved: { - threshold: 5, - predicate: event => (event as Extract).entityType === 'player', - key: 'move:player', - buildSignal: (event) => { - const e = event as Extract - return { - type: 'entity_attention', - description: `Player ${e.displayName || 'unknown'} is moving nearby`, - sourceId: e.entityId, - confidence: 0.8, - timestamp: Date.now(), - metadata: { - kind: 'player', - action: 'move', - distance: e.distance, - hasLineOfSight: e.hasLineOfSight, - displayName: e.displayName, - }, - } - }, - }, - }, - heard: { - sound: { - threshold: 5, - key: 'sound:ambient', - buildSignal: (event) => { - const e = event as Extract - return { - type: 'environmental_anomaly', - description: `Heard sound: ${e.soundId}`, - sourceId: e.soundId, - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'sound', - action: 'sound', - soundId: e.soundId, - distance: e.distance, - }, - } - }, - }, - }, - felt: { - damage_taken: { - threshold: 1, // Damage is immediately salient - key: 'felt:damage', - buildSignal: (event) => { - const e = event as Extract - const ds = (e as any).damageSource as any - const cause = typeof ds?.cause === 'string' ? ds.cause : 'unknown' - const name = typeof ds?.name === 'string' && ds.name.length > 0 ? ds.name : undefined - const distance = typeof ds?.distance === 'number' ? ds.distance : undefined - const amount = typeof (e as any).amount === 'number' ? (e as any).amount : undefined - - const details = [ - amount !== undefined ? `amount=${amount}` : null, - `cause=${cause}`, - name ? `name=${name}` : null, - distance !== undefined ? `distance=${distance.toFixed(1)}` : null, - ].filter(Boolean).join(', ') - - return { - type: 'saliency_high', - description: `Taken damage (${details}).`, - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'felt', - action: 'damage', - amount: e.amount, - damageSource: e.damageSource, - }, - } - }, - }, - item_collected: { - threshold: 3, - key: 'felt:pickup', - buildSignal: () => ({ - type: 'entity_attention', - description: 'Picked up an item', - confidence: 1.0, - timestamp: Date.now(), - metadata: { - kind: 'felt', - action: 'pickup', - }, - }), - }, - }, - system: { - system_message: { - threshold: 1, // System messages are immediately salient - key: 'system:message', - buildSignal: (event) => { - const e = event as Extract - return { - type: 'system_message', - description: e.message, - sourceId: 'system', - confidence: 1.0, - timestamp: Date.now(), - metadata: { - message: e.message, - position: e.position, - }, - } - }, - }, - }, -} diff --git a/services/minecraft/src/cognitive/perception/stage.ts b/services/minecraft/src/cognitive/perception/stage.ts deleted file mode 100644 index 2fc90ef72..000000000 --- a/services/minecraft/src/cognitive/perception/stage.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './types/stage' diff --git a/services/minecraft/src/cognitive/perception/types/raw-events.ts b/services/minecraft/src/cognitive/perception/types/raw-events.ts index e8915a53d..ca0592e1a 100644 --- a/services/minecraft/src/cognitive/perception/types/raw-events.ts +++ b/services/minecraft/src/cognitive/perception/types/raw-events.ts @@ -1,65 +1,3 @@ -import type { Vec3 } from 'vec3' - -export type PerceptionModality = 'sighted' | 'heard' | 'felt' | 'system' - -export interface RawPerceptionEventBase { - modality: PerceptionModality - timestamp: number - source: 'minecraft' - pos?: Vec3 -} - -export interface SightedEntityMovedEvent extends RawPerceptionEventBase { - modality: 'sighted' - kind: 'entity_moved' - entityType: 'player' | 'mob' - entityId: string - displayName?: string - distance: number - hasLineOfSight: boolean -} - -export interface SightedArmSwingEvent extends RawPerceptionEventBase { - modality: 'sighted' - kind: 'arm_swing' - entityType: 'player' - entityId: string - displayName?: string - distance: number - hasLineOfSight: boolean -} - -export interface SightedSneakToggleEvent extends RawPerceptionEventBase { - modality: 'sighted' - kind: 'sneak_toggle' - entityType: 'player' - entityId: string - displayName?: string - distance: number - hasLineOfSight: boolean - sneaking: boolean -} - -export type SightedEvent = SightedEntityMovedEvent | SightedArmSwingEvent | SightedSneakToggleEvent - -export interface HeardSoundEvent extends RawPerceptionEventBase { - modality: 'heard' - kind: 'sound' - soundId: string - distance: number - inferredEntityType?: 'player' | 'mob' - inferredEntityId?: string -} - -export type HeardEvent = HeardSoundEvent - -export interface FeltDamageTakenEvent extends RawPerceptionEventBase { - modality: 'felt' - kind: 'damage_taken' - amount?: number - damageSource?: DamageSourceMetadata -} - export type DamageSourceCause = | 'player' | 'mob' @@ -79,30 +17,3 @@ export interface DamageSourceMetadata { entityId?: string distance?: number } - -export interface FeltItemCollectedEvent extends RawPerceptionEventBase { - modality: 'felt' - kind: 'item_collected' - itemName: string - count?: number -} - -export type FeltEvent = FeltDamageTakenEvent | FeltItemCollectedEvent - -export interface PlayerJoinedEvent extends RawPerceptionEventBase { - modality: 'system' - kind: 'player_joined' - playerId: string - displayName?: string -} - -export interface SystemMessageEvent extends RawPerceptionEventBase { - modality: 'system' - kind: 'system_message' - message: string - position: string -} - -export type SystemEvent = PlayerJoinedEvent | SystemMessageEvent - -export type RawPerceptionEvent = SightedEvent | HeardEvent | FeltEvent | SystemEvent diff --git a/services/minecraft/src/cognitive/perception/types/signals.ts b/services/minecraft/src/cognitive/perception/types/signals.ts index 8f8aa995a..2dff1e757 100644 --- a/services/minecraft/src/cognitive/perception/types/signals.ts +++ b/services/minecraft/src/cognitive/perception/types/signals.ts @@ -1,9 +1,9 @@ export type PerceptionSignalType = | 'chat_message' - | 'entity_attention' // e.g. someone waving, teabagging + | 'entity_attention' // e.g. someone waving, punching | 'environmental_anomaly' // e.g. sudden loud sound - | 'saliency_high' // generic high saliency event - | 'social_gesture' // e.g. teabagging, waving + | 'saliency_high' // generic high-priority event (e.g. damage) + | 'social_gesture' // e.g. waving | 'social_presence' | 'system_message' // e.g. death messages, join/leave diff --git a/services/minecraft/src/cognitive/perception/types/stage.ts b/services/minecraft/src/cognitive/perception/types/stage.ts deleted file mode 100644 index 6e4b9a9bb..000000000 --- a/services/minecraft/src/cognitive/perception/types/stage.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { PerceptionFrame } from '../frame' - -export interface PerceptionStage { - name: string - tick?: (deltaMs: number) => void - handle: (frame: PerceptionFrame) => PerceptionFrame | null -} diff --git a/services/minecraft/src/debug/debug-service.ts b/services/minecraft/src/debug/debug-service.ts index cfa0c43c5..421225ef4 100644 --- a/services/minecraft/src/debug/debug-service.ts +++ b/services/minecraft/src/debug/debug-service.ts @@ -9,7 +9,6 @@ import type { ReflexStateEvent, ReplExecutionResultEvent, ReplStateEvent, - SaliencyEvent, ServerEvent, TraceEvent, } from './types' @@ -116,24 +115,6 @@ export class DebugService { this.server.broadcast(event) } - /** - * Emit a saliency snapshot - */ - public emitSaliency( - slot: number, - counters: SaliencyEvent['counters'], - ): void { - const event: ServerEvent = { - type: 'saliency', - payload: { - slot, - counters, - timestamp: Date.now(), - }, - } - this.server.broadcast(event) - } - /** * Emit a single trace event from the EventBus */ @@ -212,19 +193,6 @@ export class DebugService { case 'queue': this.server.broadcast({ type: 'queue', payload: payload as QueueEvent }) break - case 'saliency': { - // Transform the saliency data to match expected format - const data = payload as { slot?: number, counters?: SaliencyEvent['counters'] } - this.server.broadcast({ - type: 'saliency', - payload: { - slot: data.slot || 0, - counters: data.counters || [], - timestamp: Date.now(), - }, - }) - break - } case 'reflex': this.emitReflexState(payload as Omit) break diff --git a/services/minecraft/src/debug/index.ts b/services/minecraft/src/debug/index.ts index 62f4c8267..e870ab682 100644 --- a/services/minecraft/src/debug/index.ts +++ b/services/minecraft/src/debug/index.ts @@ -8,6 +8,5 @@ export type { LLMTraceEvent, LogEvent, QueueEvent, - SaliencyEvent, ServerEvent, } from './types' diff --git a/services/minecraft/src/debug/types.ts b/services/minecraft/src/debug/types.ts index c46808578..c1c89b22a 100644 --- a/services/minecraft/src/debug/types.ts +++ b/services/minecraft/src/debug/types.ts @@ -55,19 +55,6 @@ export interface QueueEvent { timestamp: number } -export interface SaliencyEvent { - slot: number - counters: Array<{ - key: string - total: number - window: number[] - triggers: number[] - lastFireSlot: number | null - lastFireTotal: number - }> - timestamp: number -} - /** * Reflex system state update */ @@ -189,7 +176,6 @@ export type ServerEvent | { type: 'llm', payload: LLMTraceEvent } | { type: 'blackboard', payload: BlackboardEvent } | { type: 'queue', payload: QueueEvent } - | { type: 'saliency', payload: SaliencyEvent } | { type: 'reflex', payload: ReflexStateEvent } | { type: 'trace', payload: TraceEvent } | { type: 'trace_batch', payload: TraceBatchEvent } diff --git a/services/minecraft/src/debug/web/app.js b/services/minecraft/src/debug/web/app.js index d2f4d4a27..68f54c00d 100644 --- a/services/minecraft/src/debug/web/app.js +++ b/services/minecraft/src/debug/web/app.js @@ -731,107 +731,6 @@ class ConversationPanel { } } -// ============================================================================= -// Saliency Panel -// ============================================================================= - -class SaliencyPanel { - constructor(client) { - this.client = client - this.data = null - this.elements = { - canvas: document.getElementById('saliency-canvas'), - labels: document.getElementById('saliency-labels'), - slot: document.getElementById('saliency-slot'), - } - this.ctx = this.elements.canvas.getContext('2d') - } - - init() { - this.client.on('saliency', data => this.update(data)) - this.client.on('connected', () => this.reset()) - } - - update(data) { - this.data = data - this.render() - } - - reset() { - this.data = null - this.render() - } - - render() { - if (!this.data || !this.data.counters) { - this.elements.labels.innerHTML = '
No data
' - return - } - - this.elements.slot.textContent = this.data.slot || 0 - - // Use all counters directly from server (fixed order) - const counters = this.data.counters - - // Render labels - this.elements.labels.innerHTML = counters.map(c => ` -
- ${escapeHtml(c.key || '')} - (${c.total || 0}) -
- `).join('') - - // Render canvas - const cols = 100 - const rows = counters.length - const cellW = 6 - const cellH = 16 - - this.elements.canvas.width = cols * cellW - this.elements.canvas.height = Math.max(1, rows) * cellH - - this.ctx.fillStyle = '#0d1117' - this.ctx.fillRect(0, 0, this.elements.canvas.width, this.elements.canvas.height) - - let maxCell = 0 - for (const counter of counters) { - for (let i = 0; i < Math.min(cols, counter.window?.length || 0); i++) { - maxCell = Math.max(maxCell, counter.window[i] || 0) - } - } - - for (let y = 0; y < rows; y++) { - const counter = counters[y] - const w = counter.window || [] - const t = counter.triggers || [] - - for (let x = 0; x < cols; x++) { - const v = x < w.length ? (w[x] || 0) : 0 - const fired = x < t.length ? (t[x] || 0) : 0 - - this.ctx.fillStyle = fired - ? 'rgba(248, 81, 73, 0.9)' - : this.colorFor(v, maxCell) - this.ctx.fillRect(x * cellW, y * cellH, cellW - 1, cellH - 1) - } - } - } - - colorFor(value, maxValue) { - // Always show a subtle color for empty cells, never fully transparent - if (value === 0) { - return 'rgba(30, 38, 50, 1)' // Dark background for empty cells - } - // Normalize value when we have data - const max = maxValue || 1 - const t = Math.min(1, value / max) - const r = Math.round(88 + 80 * t) - const g = Math.round(166 + 80 * t) - const b = Math.round(255 * t) - return `rgba(${r},${g},${b},${0.3 + 0.7 * t})` - } -} - // ============================================================================= // Tools Panel // ============================================================================= @@ -1491,7 +1390,6 @@ class LayoutManager { { id: 'splitter-v1', var: '--col-left', type: 'v' }, { id: 'splitter-h1', var: '--row-1', type: 'h' }, { id: 'splitter-h2', var: '--row-2', type: 'h' }, - { id: 'splitter-h3', var: '--row-3', type: 'h' }, ] splitters.forEach((config) => { @@ -1578,7 +1476,6 @@ class DebugApp { this.brainPanel = new BrainPanel(this.client) this.logsPanel = new LogsPanel(this.client) this.conversationPanel = new ConversationPanel(this.client) - this.saliencyPanel = new SaliencyPanel(this.client) this.timelinePanel = new TimelinePanel(this.client) this.toolsPanel = new ToolsPanel(this.client) this.replPanel = new ReplPanel(this.client) @@ -1589,7 +1486,6 @@ class DebugApp { brain: this.brainPanel, logs: this.logsPanel, conversation: this.conversationPanel, - saliency: this.saliencyPanel, timeline: this.timelinePanel, tools: this.toolsPanel, repl: this.replPanel, diff --git a/services/minecraft/src/debug/web/index.html b/services/minecraft/src/debug/web/index.html index 5a096c04b..c13db1c7a 100644 --- a/services/minecraft/src/debug/web/index.html +++ b/services/minecraft/src/debug/web/index.html @@ -207,25 +207,6 @@ - -
- - -
-
-

Saliency Heatmap

-
- Slot: 0 - -
-
-
-
- -
-
-
-
diff --git a/services/minecraft/src/debug/web/styles.css b/services/minecraft/src/debug/web/styles.css index 55de780e5..a07b070ec 100644 --- a/services/minecraft/src/debug/web/styles.css +++ b/services/minecraft/src/debug/web/styles.css @@ -182,9 +182,9 @@ button:hover, display: grid; /* Columns: Left Panel | Splitter | Right Content */ grid-template-columns: 1fr; - /* Rows: Logs | Splitter | Timeline | Splitter | LLM | Splitter | Saliency */ + /* Rows: Logs | Splitter | Timeline | Splitter | Conversation */ grid-template-rows: - minmax(var(--row-1), 1fr) 8px minmax(var(--row-2), 1fr) 8px minmax(var(--row-3), 1fr) 8px 1fr; + minmax(var(--row-1), 1fr) 8px minmax(var(--row-2), 1fr) 8px minmax(var(--row-3), 1fr); gap: 0; padding: 0; height: 100%; @@ -455,17 +455,6 @@ button:hover, min-height: 0; } -#splitter-h3 { - grid-column: 3 / 4; - grid-row: 6 / 7; -} - -#saliency-section { - grid-column: 3 / 4; - grid-row: 7 / 8; - min-height: 0; -} - /* Panel Styles */ .panel { display: flex; @@ -834,42 +823,6 @@ button:hover, 40% { transform: translateY(-3px); opacity: 0.9; } } -/* Saliency */ -.saliency-container { - display: flex; - gap: 1rem; -} - -#saliency-canvas { - border: 1px solid var(--border-color); - background-color: var(--bg-primary); -} - -.saliency-labels { - flex: 0 0 300px; - font-family: var(--font-mono); - font-size: 11px; - line-height: 16px; - overflow-y: auto; -} - -.saliency-label { - height: 16px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - border-bottom: 1px solid var(--bg-tertiary); - color: var(--text-secondary); -} - -.saliency-key { - color: var(--text-primary); -} - -.saliency-total { - color: var(--text-muted); -} - /* Utilities */ .empty-state { padding: 1rem;