nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

feat(minecraft): attempt at sliding window for salience detection

Browse Source
This commit is contained in:
Rin
2026-02-18 11:10:04 +08:00
committed by Neko Ayaka
parent f1c06e55a4
commit bc5e1f42cb
12 changed files with 604 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/minecraft/eslint.config.js
+10
View File
@@ -0,0 +1,10 @@
export default [
{
ignores: [
'docs/**',
],
rules: {
perfectionist: 'off',
},
},
]
services/minecraft/src/cognitive/conscious/brain.ts
+4 -4
View File
@@ -57,10 +57,10 @@ export class Brain {
this.log('INFO', 'Brain: Initializing...')
// Unified Perception Signal Handler
this.deps.eventManager.on<PerceptionSignal>('perception', async (event) => {
this.log('INFO', `Brain: Received perception signal: ${event.payload.type} - ${event.payload.description}`)
await this.enqueueEvent(bot, event)
})
// this.deps.eventManager.on<PerceptionSignal>('perception', async (event) => {
// this.log('INFO', `Brain: Received perception signal: ${event.payload.type} - ${event.payload.description}`)
// await this.enqueueEvent(bot, event)
// })
// Listen to Task Execution Events (Action Feedback)
this.deps.taskExecutor.on('action:completed', async ({ action, result }) => {
services/minecraft/src/cognitive/index.ts
-35
View File
@@ -8,7 +8,6 @@ import { createPerceptionFrameFromChat } from './perception/frame'
export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlugin {
let container: ReturnType<typeof createAgentContainer>
let tickHandler: ((ctx: { delta: number }) => void) | null = null
let spawnHandler: (() => void) | null = null
let started = false
@@ -22,7 +21,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug
const actionAgent = container.resolve('actionAgent')
const chatAgent = container.resolve('chatAgent')
const eventManager = container.resolve('eventManager')
const perceptionPipeline = container.resolve('perceptionPipeline')
const brain = container.resolve('brain')
const reflexManager = container.resolve('reflexManager')
@@ -50,13 +48,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug
// Initialize perception pipeline (raw events + detectors)
perceptionPipeline.init(botWithAgents)
tickHandler = ({ delta }) => {
reflexManager.tick(delta)
perceptionPipeline.tick(delta)
}
bot.onTick('tick', tickHandler)
// Set message handling via EventManager
const chatHandler = new ChatMessageHandler(bot.username)
bot.bot.on('chat', (username, message) => {
@@ -74,27 +65,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug
spawnHandler = () => startCognitive()
bot.bot.once('spawn', spawnHandler)
}
options.airiClient.onEvent('input:text:voice', (event) => {
eventManager.emit({
type: 'stimulus',
payload: {
content: event.data.transcription,
metadata: {
displayName: (event.data.discord?.guildMember as any)?.nick || (event.data.discord?.guildMember as any)?.user?.username || 'Voice Stimulus',
},
},
source: {
type: 'airi',
id: (event.data.discord?.guildMember as any)?.user?.id || 'unknown',
reply: (msg) => {
// TODO: implement Airi voice reply if needed, or just chat in MC
bot.bot.chat(msg)
},
},
timestamp: Date.now(),
})
})
},
async beforeCleanup(bot) {
@@ -119,11 +89,6 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug
}
started = false
if (tickHandler) {
bot.offTick('tick', tickHandler)
tickHandler = null
}
bot.bot.removeAllListeners('chat')
},
}
services/minecraft/src/cognitive/perception/attention-detector.test.ts
-79
View File
@@ -1,79 +0,0 @@
import { describe, expect, it } from 'vitest'
import { AttentionDetector } from './attention-detector'
function makeLogger() {
const logger: any = {
withFields: () => logger,
withError: () => logger,
log: () => { },
warn: () => { },
error: () => { },
}
return logger
}
describe('attentionDetector', () => {
it('emits punch attention after 3 arm_swing events', () => {
const emitted: any[] = []
const detector = new AttentionDetector({
logger: makeLogger(),
onAttention: payload => emitted.push(payload),
})
const base: any = {
modality: 'sighted',
kind: 'arm_swing',
entityType: 'player',
entityId: 'p1',
displayName: 'alice',
distance: 10,
hasLineOfSight: true,
timestamp: Date.now(),
source: 'minecraft',
}
detector.ingest({ ...base })
detector.ingest({ ...base })
detector.ingest({ ...base })
expect(emitted.length).toBe(1)
expect(emitted[0]).toMatchObject({
kind: 'player',
playerAction: 'punch',
playerName: 'alice',
})
})
it('emits sound attention (gated per soundId)', () => {
const emitted: any[] = []
const detector = new AttentionDetector({
logger: makeLogger(),
onAttention: payload => emitted.push(payload),
})
detector.ingest({
modality: 'heard',
kind: 'sound',
soundId: 's1',
distance: 5,
timestamp: Date.now(),
source: 'minecraft',
} as any)
detector.ingest({
modality: 'heard',
kind: 'sound',
soundId: 's1',
distance: 5,
timestamp: Date.now(),
source: 'minecraft',
} as any)
expect(emitted.length).toBe(1)
expect(emitted[0]).toMatchObject({
kind: 'player',
playerAction: 'sound',
})
})
})
services/minecraft/src/cognitive/perception/attention-detector.ts
-276
View File
@@ -1,276 +0,0 @@
import type { Logg } from '@guiiai/logg'
import type { RawPerceptionEvent } from './types/raw-events'
import type { PerceptionSignal } from './types/signals'
import { LeakyBucket } from './leaky-bucket'
export class AttentionDetector {
private readonly buckets = new Map<string, LeakyBucket>()
private lastStatsAt = 0
private emittedSinceStats: Record<string, number> = {}
private readonly movementState = new Map<
string,
{
movingSince: number
lastSeenMove: number
emitted: boolean
}
>()
private readonly dispatch: Record<string, Record<string, (event: RawPerceptionEvent) => void>> = {
sighted: {
arm_swing: event => this.onPunch(event as any),
sneak_toggle: event => this.onSneakToggle(event as any),
entity_moved: event => this.onMove(event as any),
},
heard: {
sound: event => this.onSound(event as any),
},
felt: {
damage_taken: event => this.onDamage(event as any),
item_collected: event => this.onPickup(event as any),
},
}
constructor(
private readonly deps: {
logger: Logg
onAttention: (signal: PerceptionSignal) => void
},
) { }
public tick(deltaMs: number): void {
for (const bucket of this.buckets.values()) {
bucket.tick(deltaMs)
}
const now = Date.now()
for (const [id, state] of this.movementState.entries()) {
if (now - state.lastSeenMove > 250) {
this.movementState.delete(id)
}
}
if (now - this.lastStatsAt >= 2000) {
this.deps.logger.withFields({
deltaMs,
...this.emittedSinceStats,
}).log('AttentionDetector: stats')
this.lastStatsAt = now
this.emittedSinceStats = {}
}
}
public ingest(event: RawPerceptionEvent): void {
this.dispatch[event.modality]?.[event.kind]?.(event)
}
private onPunch(event: Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'arm_swing' }>): void {
// Heuristic: 3 swings in ~1s triggers
const bucket = this.getBucket(`punch:${event.entityId}`, {
capacity: 3,
leakPerSecond: 3,
trigger: 3,
})
const { fired } = bucket.add(1)
if (!fired)
return
this.emitSignal({
type: 'entity_attention',
description: `Player ${event.displayName || 'unknown'} is punching nearby`,
sourceId: event.entityId,
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'player',
action: 'punch',
distance: event.distance,
hasLineOfSight: event.hasLineOfSight,
displayName: event.displayName,
},
})
}
private onSneakToggle(event: Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'sneak_toggle' }>): void {
// >= 4 toggles within 2s (leaky bucket approximation)
const bucket = this.getBucket(`teabag:${event.entityId}`, {
capacity: 4,
leakPerSecond: 2,
trigger: 4,
})
const { fired } = bucket.add(1)
if (!fired)
return
this.emitSignal({
type: 'entity_attention',
description: `Player ${event.displayName || 'unknown'} is teabagging (rapid sneaking)`,
sourceId: event.entityId,
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'player',
action: 'teabag',
distance: event.distance,
hasLineOfSight: event.hasLineOfSight,
displayName: event.displayName,
},
})
}
private onMove(event: Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'entity_moved' }>): void {
// Only count players for "attracting attention"
if (event.entityType !== 'player')
return
const now = Date.now()
const state = this.movementState.get(event.entityId)
if (!state) {
this.movementState.set(event.entityId, {
movingSince: now,
lastSeenMove: now,
emitted: false,
})
return
}
state.lastSeenMove = now
if (state.emitted)
return
if (now - state.movingSince < 600)
return
// Cooldown gate to avoid spamming: 1 trigger, leaks over ~3s
const bucket = this.getBucket(`move:${event.entityId}`, {
capacity: 1,
leakPerSecond: 1 / 3,
trigger: 1,
})
const { fired } = bucket.add(1)
if (!fired)
return
state.emitted = true
this.emitSignal({
type: 'entity_attention',
description: `Player ${event.displayName || 'unknown'} is moving nearby`,
sourceId: event.entityId,
confidence: 0.8,
timestamp: Date.now(),
metadata: {
kind: 'player',
action: 'move',
distance: event.distance,
hasLineOfSight: event.hasLineOfSight,
displayName: event.displayName,
},
})
}
private onSound(event: Extract<RawPerceptionEvent, { modality: 'heard', kind: 'sound' }>): void {
// Any sound within range is "interesting". Gate by soundId to prevent spam.
const bucket = this.getBucket(`sound:${event.soundId}`, {
capacity: 1,
leakPerSecond: 1, // ~1s cooldown per soundId
trigger: 1,
})
const { fired } = bucket.add(1)
if (!fired)
return
this.emitSignal({
type: 'environmental_anomaly',
description: `Heard sound: ${event.soundId}`,
sourceId: event.soundId,
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'sound',
action: 'sound',
soundId: event.soundId,
distance: event.distance,
},
})
}
private onDamage(_event: Extract<RawPerceptionEvent, { modality: 'felt', kind: 'damage_taken' }>): void {
// Self-damage is intrinsically salient; gate with small cooldown
const bucket = this.getBucket('felt:damage', {
capacity: 1,
leakPerSecond: 1 / 2,
trigger: 1,
})
const { fired } = bucket.add(1)
if (!fired)
return
this.emitSignal({
type: 'saliency_high',
description: 'Taken damage!',
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'felt',
action: 'damage',
},
})
}
private onPickup(_event: Extract<RawPerceptionEvent, { modality: 'felt', kind: 'item_collected' }>): void {
// Item pickup can be spammy (e.g. farms); apply a small cooldown
const bucket = this.getBucket('felt:pickup', {
capacity: 1,
leakPerSecond: 1, // ~1s cooldown
trigger: 1,
})
const { fired } = bucket.add(1)
if (!fired)
return
this.emitSignal({
type: 'entity_attention',
description: 'Picked up an item',
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'felt',
action: 'pickup',
},
})
}
private emitSignal(signal: PerceptionSignal): void {
const key = `emit.${signal.type}.${signal.metadata.action || 'unknown'}`
this.emittedSinceStats[key] = (this.emittedSinceStats[key] ?? 0) + 1
this.deps.logger.withFields({
type: signal.type,
desc: signal.description,
meta: signal.metadata,
}).log('AttentionDetector: emit')
this.deps.onAttention(signal)
}
private getBucket(key: string, config: { capacity: number, leakPerSecond: number, trigger: number }): LeakyBucket {
const existing = this.buckets.get(key)
if (existing)
return existing
const created = new LeakyBucket(config)
this.buckets.set(key, created)
return created
}
}
services/minecraft/src/cognitive/perception/pipeline.e2e.test.ts
-174
View File
@@ -1,174 +0,0 @@
import type { MineflayerWithAgents } from '../types'
import { EventEmitter } from 'node:events'
import { Vec3 } from 'vec3'
import { describe, expect, it, vi } from 'vitest'
import { EventManager } from './event-manager'
import { createPerceptionFrameFromChat } from './frame'
import { PerceptionPipeline } from './pipeline'
function makeLogger() {
const logger: any = {
withFields: () => logger,
withError: () => logger,
log: () => { },
warn: () => { },
error: () => { },
}
return logger
}
function makeBotWithAgents() {
const emitter = new EventEmitter()
const bot: any = emitter
bot.entity = {
position: new Vec3(0, 0, 0),
height: 1.8,
}
bot.health = 20
bot.food = 20
bot.time = { isDay: true }
bot.isRaining = false
bot.players = {}
const mineflayerWithAgents = {
bot,
username: 'test-bot',
action: {} as any,
chat: {} as any,
planning: {} as any,
} satisfies Partial<MineflayerWithAgents>
return mineflayerWithAgents as MineflayerWithAgents
}
describe('perceptionPipeline (e2e)', () => {
it('mineflayer events flow: collector -> normalizer -> attention -> router -> EventManager', () => {
const logger = makeLogger()
const eventManager = new EventManager()
const emitSpy = vi.spyOn(eventManager, 'emit')
const pipeline = new PerceptionPipeline({ eventManager, logger })
const botWithAgents = makeBotWithAgents()
pipeline.init(botWithAgents)
const entity: any = {
type: 'player',
id: 1,
username: 'alice',
position: new Vec3(1, 0, 1),
metadata: [],
}
// 3 swings => punch attention
botWithAgents.bot.emit('entitySwingArm', entity)
botWithAgents.bot.emit('entitySwingArm', entity)
botWithAgents.bot.emit('entitySwingArm', entity)
pipeline.tick(0)
const perceptionEvents = emitSpy.mock.calls
.map(c => c[0])
.filter(e => e.type === 'perception')
expect(perceptionEvents.length).toBeGreaterThanOrEqual(1)
// Updated assertion for PerceptionSignal
const signal = (perceptionEvents[0] as any).payload
expect(signal.type).toBe('entity_attention')
expect(signal.metadata).toMatchObject({
kind: 'player',
action: 'punch',
displayName: 'alice',
})
pipeline.destroy()
})
it('router emits perception signal for chat frames ingested into pipeline', () => {
const logger = makeLogger()
const eventManager = new EventManager()
const emitSpy = vi.spyOn(eventManager, 'emit')
const pipeline = new PerceptionPipeline({ eventManager, logger })
const botWithAgents = makeBotWithAgents()
pipeline.init(botWithAgents)
pipeline.ingest(createPerceptionFrameFromChat('alice', 'hi'))
pipeline.tick(0)
// Should now be a 'perception' event
const perceptionEvents = emitSpy.mock.calls
.map(c => c[0])
.filter(e => e.type === 'perception')
expect(perceptionEvents.length).toBe(1)
const signal = (perceptionEvents[0] as any).payload
expect(signal.type).toBe('chat_message')
expect(signal.description).toContain('alice')
expect(signal.description).toContain('hi')
expect(signal.metadata).toMatchObject({
username: 'alice',
message: 'hi',
})
pipeline.destroy()
})
it('normalizer drops throttled entity_moved events (e2e)', () => {
vi.useFakeTimers()
try {
const logger = makeLogger()
const eventManager = new EventManager()
const emitSpy = vi.spyOn(eventManager, 'emit')
const pipeline = new PerceptionPipeline({ eventManager, logger })
const botWithAgents = makeBotWithAgents()
pipeline.init(botWithAgents)
const entity: any = {
type: 'player',
id: 1,
username: 'alice',
position: new Vec3(1, 0, 1),
// 0th metadata is flags in collector, keep stable
metadata: [0],
}
// movement attention requires sustained movement; we only assert no errors and that
// throttling doesn't allow duplicates through attention detector.
vi.setSystemTime(new Date(0))
botWithAgents.bot.emit('entityMoved', entity)
vi.setSystemTime(new Date(50))
botWithAgents.bot.emit('entityMoved', entity)
vi.setSystemTime(new Date(200))
botWithAgents.bot.emit('entityMoved', entity)
pipeline.tick(0)
// At most 2 move raws should make it past normalizer (t=0 and t=200)
// We can't observe raw frames directly here, so we at least ensure we didn't emit
// an absurd number of perception events.
const perceptionEvents = emitSpy.mock.calls
.map(c => c[0])
.filter(e => e.type === 'perception')
expect(perceptionEvents.length).toBeLessThanOrEqual(2)
pipeline.destroy()
}
finally {
vi.useRealTimers()
}
})
})
services/minecraft/src/cognitive/perception/pipeline.ts
+29 -58
View File
@@ -6,14 +6,13 @@ import type { PerceptionFrame } from './frame'
import type { PerceptionSignal } from './types/signals'
import type { PerceptionStage } from './types/stage'
import { AttentionDetector } from './attention-detector'
import { DebugService } from '../../debug-server'
import { SaliencyDetector } from './saliency-detector'
import { createPerceptionFrameFromRawEvent } from './frame'
import { MineflayerPerceptionCollector } from './mineflayer-perception-collector'
import { RawEventBuffer } from './raw-event-buffer'
export class PerceptionPipeline {
private readonly buffer = new RawEventBuffer()
private readonly detector: AttentionDetector
private readonly detector: SaliencyDetector
private collector: MineflayerPerceptionCollector | null = null
private initialized = false
@@ -21,9 +20,7 @@ export class PerceptionPipeline {
private currentFrame: PerceptionFrame | null = null
private lastStatsAt = 0
private collectedSinceStats = 0
private processedSinceStats = 0
private saliencyEmitTimer: ReturnType<typeof setInterval> | null = null
constructor(
private readonly deps: {
@@ -31,7 +28,7 @@ export class PerceptionPipeline {
logger: Logg
},
) {
this.detector = new AttentionDetector({
this.detector = new SaliencyDetector({
logger: this.deps.logger,
onAttention: (signal) => {
// This is only called synchronously while we're handling a specific frame.
@@ -46,9 +43,6 @@ export class PerceptionPipeline {
this.stages = [
{
name: 'attention',
tick: (deltaMs) => {
this.detector.tick(deltaMs)
},
handle: (frame) => {
if (frame.kind !== 'world_raw')
return frame
@@ -118,12 +112,16 @@ export class PerceptionPipeline {
public init(bot: MineflayerWithAgents): void {
this.initialized = true
this.lastStatsAt = Date.now()
this.collectedSinceStats = 0
this.processedSinceStats = 0
this.deps.logger.withFields({ maxDistance: 32 }).log('PerceptionPipeline: init')
this.detector.start()
this.saliencyEmitTimer = setInterval(() => {
if (!this.initialized)
return
DebugService.getInstance().emit('saliency', this.detector.getDebugSnapshot({ maxKeys: 30 }))
}, 100)
this.collector = new MineflayerPerceptionCollector({
logger: this.deps.logger,
emitRaw: (event) => {
@@ -138,58 +136,31 @@ export class PerceptionPipeline {
this.deps.logger.log('PerceptionPipeline: destroy')
this.collector?.destroy()
this.collector = null
this.buffer.clear()
if (this.saliencyEmitTimer) {
clearInterval(this.saliencyEmitTimer)
this.saliencyEmitTimer = null
}
this.detector.stop()
this.initialized = false
}
public ingest(frame: PerceptionFrame): void {
if (!this.initialized)
return
this.buffer.push(frame)
this.collectedSinceStats++
}
public tick(deltaMs: number): void {
if (!this.initialized)
return
const startedAt = Date.now()
let current: PerceptionFrame | null = frame
for (const stage of this.stages) {
stage.tick?.(deltaMs)
}
const frames = this.buffer.drain()
this.processedSinceStats += frames.length
for (const frame of frames) {
let current: PerceptionFrame | null = frame
for (const stage of this.stages) {
if (!current)
break
try {
current = stage.handle(current)
}
catch (err) {
this.deps.logger.withError(err as Error).error('PerceptionPipeline: stage error')
break
}
if (!current)
break
try {
current = stage.handle(current)
}
catch (err) {
this.deps.logger.withError(err as Error).error('PerceptionPipeline: stage error')
break
}
}
const now = Date.now()
if (now - this.lastStatsAt >= 2000) {
this.deps.logger.withFields({
deltaMs,
tickCostMs: now - startedAt,
queueDepth: this.buffer.size(),
drained: frames.length,
collected: this.collectedSinceStats,
processed: this.processedSinceStats,
}).log('PerceptionPipeline: stats')
this.lastStatsAt = now
this.collectedSinceStats = 0
this.processedSinceStats = 0
}
}
}
services/minecraft/src/cognitive/perception/saliency-detector.ts
+232
View File
@@ -0,0 +1,232 @@
import type { Logg } from '@guiiai/logg'
import { DEFAULT_THRESHOLD, DEFAULT_WINDOW_TICKS, SALIENCY_RULES, type SaliencyRuleBook } from './saliency-rules'
import type { RawPerceptionEvent } from './types/raw-events'
import type { PerceptionSignal } from './types/signals'
type WindowCounter = {
windowTicks: number
head: number
counts: number[]
triggers: number[]
total: number
lastEventSlot: number
lastFireSlot: number | null
lastFireTotal: number
}
export class SaliencyDetector {
private readonly counters = new Map<string, WindowCounter>()
private currentSlot = 0
private timer: ReturnType<typeof setInterval> | null = null
private readonly slotMs = 20
private lastStatsAt = 0
private emittedSinceStats: Record<string, number> = {}
constructor(
private readonly deps: {
logger: Logg
onAttention: (signal: PerceptionSignal) => void
rules?: SaliencyRuleBook
windowTicks?: number
threshold?: number
},
) { }
public start(): void {
if (this.timer)
return
this.timer = setInterval(() => {
this.currentSlot += 1
this.advanceWindows()
const now = Date.now()
if (now - this.lastStatsAt >= 2000) {
this.lastStatsAt = now
this.emittedSinceStats = {}
}
}, this.slotMs)
}
public stop(): void {
if (!this.timer)
return
clearInterval(this.timer)
this.timer = null
}
public ingest(event: RawPerceptionEvent): void {
const rule = this.lookupRule(event)
if (!rule)
return
if (rule.predicate && !rule.predicate(event))
return
const tick = this.currentSlot
const windowTicks = rule.windowTicks ?? this.deps.windowTicks ?? DEFAULT_WINDOW_TICKS
const threshold = rule.threshold ?? this.deps.threshold ?? DEFAULT_THRESHOLD
const key = rule.key(event)
const counter = this.getOrCreateCounter(key, windowTicks)
counter.counts[counter.head] = (counter.counts[counter.head] ?? 0) + 1
counter.total += 1
counter.lastEventSlot = tick
if (counter.total >= threshold) {
counter.lastFireSlot = tick
counter.lastFireTotal = counter.total
counter.triggers[counter.head] = 1
this.resetCounter(counter)
const signal = rule.buildSignal(event)
this.emitSignal(signal)
}
}
public getDebugSnapshot(options?: { maxKeys?: number }): {
slot: number
keys: Array<{
key: string
total: number
windowTicks: number
window: number[]
triggers: number[]
lastFireSlot: number | null
lastFireTotal: number
}>
} {
const maxKeys = options?.maxKeys ?? 30
const rows = Array.from(this.counters.entries()).map(([key, counter]) => {
const window = this.exportWindow(counter)
const triggers = this.exportTriggers(counter)
const triggerSum = triggers.reduce((acc, v) => acc + (v ? 1 : 0), 0)
const firedRecently = counter.lastFireSlot !== null && (this.currentSlot - counter.lastFireSlot) <= counter.windowTicks
return {
key,
total: counter.total,
windowTicks: counter.windowTicks,
window,
triggers,
lastFireSlot: counter.lastFireSlot,
lastFireTotal: counter.lastFireTotal,
_triggerSum: triggerSum,
_firedRecently: firedRecently,
}
})
// Ensure keys with triggers/recent fires stay visible even if total was reset.
rows.sort((a, b) => {
if (a._triggerSum !== b._triggerSum)
return b._triggerSum - a._triggerSum
const af = a._firedRecently ? 1 : 0
const bf = b._firedRecently ? 1 : 0
if (af !== bf)
return bf - af
const at = a.lastFireSlot ?? -1
const bt = b.lastFireSlot ?? -1
if (at !== bt)
return bt - at
return b.total - a.total
})
return {
slot: this.currentSlot,
keys: rows.slice(0, maxKeys).map(({ _triggerSum: _ts, _firedRecently: _fr, ...row }) => row),
}
}
private advanceWindows(): void {
for (const [key, counter] of this.counters.entries()) {
counter.head = (counter.head + 1) % counter.windowTicks
const expired = counter.counts[counter.head] ?? 0
if (expired > 0) {
counter.total = Math.max(0, counter.total - expired)
counter.counts[counter.head] = 0
}
else {
counter.counts[counter.head] = 0
}
counter.triggers[counter.head] = 0
if (counter.total === 0 && this.currentSlot - counter.lastEventSlot >= counter.windowTicks) {
this.counters.delete(key)
}
}
}
private getOrCreateCounter(key: string, windowTicks: number): WindowCounter {
const existing = this.counters.get(key)
if (existing && existing.windowTicks === windowTicks)
return existing
const created: WindowCounter = {
windowTicks,
head: 0,
counts: Array.from({ length: windowTicks }, () => 0),
triggers: Array.from({ length: windowTicks }, () => 0),
total: 0,
lastEventSlot: this.currentSlot,
lastFireSlot: null,
lastFireTotal: 0,
}
this.counters.set(key, created)
return created
}
private resetCounter(counter: WindowCounter): void {
counter.total = 0
counter.counts.fill(0)
}
private exportWindow(counter: WindowCounter): number[] {
const w = counter.windowTicks
const out = new Array<number>(w)
// Oldest -> newest. The newest bucket is at `head`.
for (let i = 0; i < w; i++) {
const idx = (counter.head + 1 + i) % w
out[i] = counter.counts[idx] ?? 0
}
return out
}
private exportTriggers(counter: WindowCounter): number[] {
const w = counter.windowTicks
const out = new Array<number>(w)
// Oldest -> newest. The newest bucket is at `head`.
for (let i = 0; i < w; i++) {
const idx = (counter.head + 1 + i) % w
out[i] = counter.triggers[idx] ?? 0
}
return out
}
private lookupRule(event: RawPerceptionEvent) {
const rules = this.deps.rules ?? SALIENCY_RULES
return rules[event.modality]?.[event.kind]
}
private emitSignal(signal: PerceptionSignal): void {
const key = `emit.${signal.type}.${signal.metadata.action || 'unknown'}`
this.emittedSinceStats[key] = (this.emittedSinceStats[key] ?? 0) + 1
this.deps.logger.withFields({
type: signal.type,
desc: signal.description,
meta: signal.metadata,
}).log('SaliencyDetector: emit')
this.deps.onAttention(signal)
}
}
services/minecraft/src/cognitive/perception/saliency-rules.ts
+149
View File
@@ -0,0 +1,149 @@
import type { RawPerceptionEvent } from './types/raw-events'
import type { PerceptionSignal } from './types/signals'
export interface SaliencyRule<E extends RawPerceptionEvent = RawPerceptionEvent> {
/**
* How many occurrences within the window are required before emitting.
* Defaults to 5.
*/
threshold?: number
/**
* Window size in ticks. Defaults to 100 ticks (~5s).
*/
windowTicks?: number
/**
* Optional predicate to gate the rule.
*/
predicate?: (event: E) => boolean
/**
* Counter key used to bucket occurrences.
*/
key: (event: E) => string
/**
* Builds the PerceptionSignal when the rule fires.
*/
buildSignal: (event: E) => PerceptionSignal
}
export type SaliencyRuleBook = Partial<Record<
RawPerceptionEvent['modality'],
Record<string, SaliencyRule>
>>
export const DEFAULT_WINDOW_TICKS = 100
export const DEFAULT_THRESHOLD = 5
export const SALIENCY_RULES: SaliencyRuleBook = {
sighted: {
arm_swing: {
key: event => `punch:${(event as Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'arm_swing' }>).entityId}`,
buildSignal: (event) => {
const e = event as Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'arm_swing' }>
return {
type: 'entity_attention',
description: `Player ${e.displayName || 'unknown'} is punching nearby`,
sourceId: e.entityId,
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'player',
action: 'punch',
distance: e.distance,
hasLineOfSight: e.hasLineOfSight,
displayName: e.displayName,
},
}
},
},
sneak_toggle: {
key: event => `teabag:${(event as Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'sneak_toggle' }>).entityId}`,
buildSignal: (event) => {
const e = event as Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'sneak_toggle' }>
return {
type: 'entity_attention',
description: `Player ${e.displayName || 'unknown'} is teabagging (rapid sneaking)`,
sourceId: e.entityId,
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'player',
action: 'teabag',
distance: e.distance,
hasLineOfSight: e.hasLineOfSight,
displayName: e.displayName,
},
}
},
},
entity_moved: {
predicate: event => (event as Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'entity_moved' }>).entityType === 'player',
key: event => `move:${(event as Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'entity_moved' }>).entityId}`,
buildSignal: (event) => {
const e = event as Extract<RawPerceptionEvent, { modality: 'sighted', kind: 'entity_moved' }>
return {
type: 'entity_attention',
description: `Player ${e.displayName || 'unknown'} is moving nearby`,
sourceId: e.entityId,
confidence: 0.8,
timestamp: Date.now(),
metadata: {
kind: 'player',
action: 'move',
distance: e.distance,
hasLineOfSight: e.hasLineOfSight,
displayName: e.displayName,
},
}
},
},
},
heard: {
sound: {
key: event => `sound:${(event as Extract<RawPerceptionEvent, { modality: 'heard', kind: 'sound' }>).soundId}`,
buildSignal: (event) => {
const e = event as Extract<RawPerceptionEvent, { modality: 'heard', kind: 'sound' }>
return {
type: 'environmental_anomaly',
description: `Heard sound: ${e.soundId}`,
sourceId: e.soundId,
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'sound',
action: 'sound',
soundId: e.soundId,
distance: e.distance,
},
}
},
},
},
felt: {
damage_taken: {
key: () => 'felt:damage',
buildSignal: (_event) => ({
type: 'saliency_high',
description: 'Taken damage!',
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'felt',
action: 'damage',
},
}),
},
item_collected: {
key: () => 'felt:pickup',
buildSignal: (_event) => ({
type: 'entity_attention',
description: 'Picked up an item',
confidence: 1.0,
timestamp: Date.now(),
metadata: {
kind: 'felt',
action: 'pickup',
},
}),
},
},
}
services/minecraft/src/cognitive/perception/signals.test.ts
-88
View File
@@ -1,88 +0,0 @@
import type { RawPerceptionEvent } from './types/raw-events'
import { describe, expect, it, vi } from 'vitest'
import { AttentionDetector } from './attention-detector'
function makeLogger() {
const logger: any = {
withFields: () => logger,
withError: () => logger,
log: () => { },
warn: () => { },
error: () => { },
}
return logger
}
describe('attentionDetector (Signals)', () => {
it('emits entity_attention signal for punch', () => {
const onAttention = vi.fn()
const detector = new AttentionDetector({ logger: makeLogger(), onAttention })
const event: RawPerceptionEvent = {
modality: 'sighted',
kind: 'arm_swing',
entityId: '123',
entityType: 'player',
displayName: 'Steve',
distance: 5,
hasLineOfSight: true,
timestamp: Date.now(),
pos: { x: 0, y: 0, z: 0 } as any,
source: 'minecraft',
} as any
// Trigger 3 times
detector.ingest(event)
detector.ingest(event)
detector.ingest(event)
expect(onAttention).toHaveBeenCalledTimes(1)
const signal = onAttention.mock.calls[0][0]
expect(signal).toMatchObject({
type: 'entity_attention',
sourceId: '123',
confidence: 1.0,
metadata: {
kind: 'player',
action: 'punch',
displayName: 'Steve',
},
})
expect(signal.description).toContain('Steve')
expect(signal.description).toContain('punching')
})
it('emits environmental_anomaly signal for sound', () => {
const onAttention = vi.fn()
const detector = new AttentionDetector({ logger: makeLogger(), onAttention })
const event: RawPerceptionEvent = {
modality: 'heard',
kind: 'sound',
soundId: 'entity.zombie.ambient',
distance: 10,
timestamp: Date.now(),
source: 'minecraft',
pos: { x: 0, y: 0, z: 0 } as any,
} as any
detector.ingest(event)
expect(onAttention).toHaveBeenCalledTimes(1)
const signal = onAttention.mock.calls[0][0]
expect(signal).toMatchObject({
type: 'environmental_anomaly',
sourceId: 'entity.zombie.ambient',
confidence: 1.0,
metadata: {
kind: 'sound',
action: 'sound',
soundId: 'entity.zombie.ambient',
},
})
})
})
services/minecraft/src/cognitive/reflex/reflex-manager.ts
+3 -14
View File
@@ -39,13 +39,6 @@ export class ReflexManager {
this.bot = null
}
public tick(deltaMs: number): void {
if (!this.bot)
return
this.runtime.tick(this.bot, deltaMs)
}
public getContextSnapshot(): ReflexContextState {
return this.runtime.getContext().getSnapshot()
}
@@ -56,13 +49,10 @@ export class ReflexManager {
return
const signal = event.payload
// Only care about chat messages for now for social context
if (signal.type !== 'chat_message')
return
const message = `Signal triggered: ${signal.type} - ${signal.description}`
bot.bot.chat(message)
const now = Date.now()
const message = signal.metadata.message || signal.description
this.runtime.getContext().updateNow(now)
this.runtime.getContext().updateSocial({
lastSpeaker: event.source.id,
@@ -71,8 +61,7 @@ export class ReflexManager {
})
const behaviorId = this.runtime.tick(bot, 0)
if (behaviorId) {
if (behaviorId)
event.handled = true
}
}
}
services/minecraft/src/web/dashboard.html
+177 -1
View File
@@ -66,6 +66,36 @@
flex-direction: column;
}
.heatmap-wrap {
display: flex;
gap: 1rem;
align-items: flex-start;
}
.heatmap-legend {
color: #888;
font-size: 0.85em;
margin-top: 0.5rem;
}
.heatmap-labels {
width: 260px;
max-width: 260px;
overflow: hidden;
font-family: 'Consolas', 'Monaco', monospace;
font-size: 12px;
line-height: 16px;
color: #aaa;
}
.heatmap-label {
height: 16px;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
border-bottom: 1px solid #222;
}
.nav-item {
padding: 1rem;
cursor: pointer;
@@ -253,6 +283,7 @@
<div class="nav-item" onclick="switchTab('logs')">Logs</div>
<div class="nav-item" onclick="switchTab('llm')">LLM Traces</div>
<div class="nav-item" onclick="switchTab('blackboard')">Blackboard</div>
<div class="nav-item" onclick="switchTab('saliency')">Saliency</div>
</nav>
<div id="overview-panel" class="content-panel active">
@@ -302,11 +333,39 @@
</div>
<div id="llm-list" class="log-container"></div>
</div>
<div id="saliency-panel" class="content-panel">
<div class="toolbar" style="align-items: center;">
<h3 style="margin: 0;">Saliency Window</h3>
<div style="margin-left: auto; display: flex; gap: 0.75rem; align-items: center; color:#aaa; font-size: 0.9em;">
<span>Top</span>
<input id="saliency-topn" type="number" min="1" max="100" value="30" style="width: 64px;" />
<span>Slot</span>
<span id="saliency-slot" class="badge">0</span>
</div>
</div>
<div class="log-container" style="padding: 1rem;">
<div class="heatmap-wrap">
<div>
<canvas id="saliency-canvas" width="800" height="480"
style="border: 1px solid #222; background:#0f0f0f;"></canvas>
<div class="heatmap-legend">Columns: oldest → newest (100 slots). Rows: counter keys (top-N by total).</div>
</div>
<div>
<div class="section-title" style="margin-top:0;">Keys</div>
<div id="saliency-labels" class="heatmap-labels"></div>
</div>
</div>
</div>
</div>
</main>
<script>
var autoScroll = true;
var sseClient = null;
var latestSaliency = null;
var saliencyKeyOrder = [];
var saliencyKeyMap = new Map();
const statusDot = document.getElementById('connection-status');
const statusText = document.getElementById('status-text');
@@ -526,10 +585,127 @@
updateBlackboard(JSON.parse(e.data));
} catch (e) { console.error(e); }
});
sseClient.addEventListener('saliency', (e) => {
try {
latestSaliency = JSON.parse(e.data);
ingestSaliency(latestSaliency);
renderSaliency();
} catch (e) { console.error(e); }
});
}
function ingestSaliency(snapshot) {
if (!snapshot || !snapshot.keys) return;
const slot = snapshot.slot || 0;
const TTL_SLOTS = 300;
for (const row of snapshot.keys) {
const key = row.key;
if (!key) continue;
if (!saliencyKeyMap.has(key)) {
saliencyKeyOrder.push(key);
}
saliencyKeyMap.set(key, {
key,
total: row.total || 0,
windowTicks: row.windowTicks || 100,
window: Array.isArray(row.window) ? row.window : [],
triggers: Array.isArray(row.triggers) ? row.triggers : [],
lastFireSlot: (row.lastFireSlot === null || row.lastFireSlot === undefined) ? null : row.lastFireSlot,
lastFireTotal: row.lastFireTotal || 0,
lastSeenSlot: slot,
});
}
// Cleanup stale keys to avoid unbounded growth
saliencyKeyOrder = saliencyKeyOrder.filter((key) => {
const item = saliencyKeyMap.get(key);
if (!item) return false;
if (slot - item.lastSeenSlot > TTL_SLOTS) {
saliencyKeyMap.delete(key);
return false;
}
return true;
});
}
function colorFor(value, maxValue) {
if (!maxValue || maxValue <= 0) return 'rgba(0,0,0,0)';
const t = Math.max(0, Math.min(1, value / maxValue));
// dark -> bright blue
const r = Math.round(20 + 30 * t);
const g = Math.round(40 + 80 * t);
const b = Math.round(80 + 170 * t);
const a = 0.15 + 0.85 * t;
return `rgba(${r},${g},${b},${a})`;
}
function renderSaliency() {
if (!latestSaliency) return;
const topNInput = document.getElementById('saliency-topn');
const topN = Math.max(1, Math.min(100, parseInt(topNInput.value || '30', 10)));
const tickBadge = document.getElementById('saliency-slot');
tickBadge.textContent = String(latestSaliency.slot || 0);
const order = saliencyKeyOrder.slice(Math.max(0, saliencyKeyOrder.length - topN));
const keys = order.map(k => saliencyKeyMap.get(k)).filter(Boolean);
const labels = document.getElementById('saliency-labels');
labels.innerHTML = keys.map(k => {
const fire = (k.lastFireSlot !== null && k.lastFireSlot !== undefined)
? ` <span style="color:#666">fire:${k.lastFireTotal || 0}@${k.lastFireSlot}</span>`
: '';
return `<div class="heatmap-label">${(k.key || '')} <span style="color:#666">(${k.total || 0})</span>${fire}</div>`;
}).join('') || '<div style="color:#666">No data</div>';
const canvas = document.getElementById('saliency-canvas');
const ctx = canvas.getContext('2d');
const cols = 100;
const rows = keys.length;
const cellW = 6;
const cellH = 16;
canvas.width = cols * cellW;
canvas.height = Math.max(1, rows) * cellH;
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = '#0f0f0f';
ctx.fillRect(0, 0, canvas.width, canvas.height);
let maxCell = 0;
for (const row of keys) {
const w = row.window || [];
for (let i = 0; i < Math.min(cols, w.length); i++) {
const v = w[i] || 0;
if (v > maxCell) maxCell = v;
}
}
for (let y = 0; y < rows; y++) {
const row = keys[y];
const w = row.window || [];
const t = row.triggers || [];
for (let x = 0; x < cols; x++) {
const v = (x < w.length) ? (w[x] || 0) : 0;
const fired = (x < t.length) ? (t[x] || 0) : 0;
ctx.fillStyle = fired ? 'rgba(255, 80, 80, 0.95)' : colorFor(v, maxCell);
ctx.fillRect(x * cellW, y * cellH, cellW - 1, cellH - 1);
}
}
}
document.getElementById('saliency-topn').addEventListener('change', () => {
renderSaliency();
});
connectSSE();
</script>
</body>
</html>
</html>