From 585ff689bcb6bb08a162a0a751fd6b676382144a Mon Sep 17 00:00:00 2001 From: Rin Date: Sun, 11 Jan 2026 00:24:38 +0800 Subject: [PATCH] feat(minecraft): implement reflex layer and related refactors --- .../minecraft/src/cognitive/action/types.ts | 22 +-- .../src/cognitive/action/types/index.ts | 21 +++ .../src/cognitive/conscious/blackboard.ts | 57 ++------ .../src/cognitive/conscious/brain.ts | 20 +-- .../src/cognitive/conscious/context-view.ts | 19 +++ .../conscious/prompts/brain-prompt.ts | 4 +- services/minecraft/src/cognitive/container.ts | 8 +- services/minecraft/src/cognitive/index.ts | 4 + .../perception/attention-detector.ts | 2 +- .../src/cognitive/perception/frame.ts | 2 +- .../mineflayer-perception-collector.ts | 2 +- .../cognitive/perception/normalizer-stage.ts | 4 +- .../src/cognitive/perception/pipeline.ts | 2 +- .../src/cognitive/perception/raw-events.ts | 75 +--------- .../src/cognitive/perception/stage.ts | 8 +- .../cognitive/perception/types/raw-events.ts | 74 ++++++++++ .../src/cognitive/perception/types/stage.ts | 7 + .../src/cognitive/reflex/behavior.ts | 1 + .../cognitive/reflex/behaviors/greeting.ts | 40 ++++++ .../minecraft/src/cognitive/reflex/context.ts | 110 +++++++++++++++ .../minecraft/src/cognitive/reflex/modes.ts | 13 ++ .../cognitive/reflex/reflex-manager.test.ts | 61 +++++++++ .../src/cognitive/reflex/reflex-manager.ts | 69 +++++++--- .../minecraft/src/cognitive/reflex/runtime.ts | 129 ++++++++++++++++++ .../src/cognitive/reflex/types/behavior.ts | 22 +++ services/minecraft/src/cognitive/types.ts | 62 +-------- .../minecraft/src/cognitive/types/index.ts | 61 +++++++++ 27 files changed, 650 insertions(+), 249 deletions(-) create mode 100644 services/minecraft/src/cognitive/action/types/index.ts create mode 100644 services/minecraft/src/cognitive/conscious/context-view.ts create mode 100644 services/minecraft/src/cognitive/perception/types/raw-events.ts create mode 100644 services/minecraft/src/cognitive/perception/types/stage.ts create mode 100644 services/minecraft/src/cognitive/reflex/behavior.ts create mode 100644 services/minecraft/src/cognitive/reflex/behaviors/greeting.ts create mode 100644 services/minecraft/src/cognitive/reflex/context.ts create mode 100644 services/minecraft/src/cognitive/reflex/modes.ts create mode 100644 services/minecraft/src/cognitive/reflex/reflex-manager.test.ts create mode 100644 services/minecraft/src/cognitive/reflex/runtime.ts create mode 100644 services/minecraft/src/cognitive/reflex/types/behavior.ts create mode 100644 services/minecraft/src/cognitive/types/index.ts diff --git a/services/minecraft/src/cognitive/action/types.ts b/services/minecraft/src/cognitive/action/types.ts index 48d58fd63..679c9fc91 100644 --- a/services/minecraft/src/cognitive/action/types.ts +++ b/services/minecraft/src/cognitive/action/types.ts @@ -1,21 +1 @@ -import type { PlanStep } from '../../agents/planning/adapter' - -export type ActionType = 'physical' | 'chat' - -export interface BaseActionInstruction { - type: ActionType - description?: string - require_feedback?: boolean -} - -export interface PhysicalActionInstruction extends BaseActionInstruction { - type: 'physical' - step: PlanStep -} - -export interface ChatActionInstruction extends BaseActionInstruction { - type: 'chat' - message: string -} - -export type ActionInstruction = PhysicalActionInstruction | ChatActionInstruction +export * from './types/index' diff --git a/services/minecraft/src/cognitive/action/types/index.ts b/services/minecraft/src/cognitive/action/types/index.ts new file mode 100644 index 000000000..db96c591d --- /dev/null +++ b/services/minecraft/src/cognitive/action/types/index.ts @@ -0,0 +1,21 @@ +import type { PlanStep } from '../../../agents/planning/adapter' + +export type ActionType = 'physical' | 'chat' + +export interface BaseActionInstruction { + type: ActionType + description?: string + require_feedback?: boolean +} + +export interface PhysicalActionInstruction extends BaseActionInstruction { + type: 'physical' + step: PlanStep +} + +export interface ChatActionInstruction extends BaseActionInstruction { + type: 'chat' + message: string +} + +export type ActionInstruction = PhysicalActionInstruction | ChatActionInstruction diff --git a/services/minecraft/src/cognitive/conscious/blackboard.ts b/services/minecraft/src/cognitive/conscious/blackboard.ts index dca34c2f0..9602da05e 100644 --- a/services/minecraft/src/cognitive/conscious/blackboard.ts +++ b/services/minecraft/src/cognitive/conscious/blackboard.ts @@ -1,28 +1,13 @@ -import type { Vec3 } from 'vec3' - -export interface SelfState { - status: 'idle' | 'moving' | 'working' | 'chatting' | 'busy' - location: Vec3 | null - holding: string | null - health: number - food: number - oxygen: number -} - -export interface EnvironmentState { - time: string // 'day' | 'night' | 'sunset' | 'sunrise' - weather: 'clear' | 'rain' | 'thunder' - nearbyPlayers: string[] - nearbyEntities: string[] // significant entities (mobs, dropped items of interest) - lightLevel: number +export interface ContextViewState { + selfSummary: string + environmentSummary: string } export interface BlackboardState { currentGoal: string currentThought: string executionStrategy: string - self: SelfState - environment: EnvironmentState + contextView: ContextViewState } export class Blackboard { @@ -33,20 +18,9 @@ export class Blackboard { currentGoal: 'Idle', currentThought: 'I am waiting for something to happen.', executionStrategy: 'Observe surroundings.', - self: { - status: 'idle', - location: null, - holding: null, - health: 20, - food: 20, - oxygen: 20, - }, - environment: { - time: 'day', - weather: 'clear', - nearbyPlayers: [], - nearbyEntities: [], - lightLevel: 15, + contextView: { + selfSummary: 'Unknown', + environmentSummary: 'Unknown', }, } } @@ -55,29 +29,22 @@ export class Blackboard { public get goal(): string { return this._state.currentGoal } public get thought(): string { return this._state.currentThought } public get strategy(): string { return this._state.executionStrategy } - public get self(): SelfState { return this._state.self } - public get environment(): EnvironmentState { return this._state.environment } + public get selfSummary(): string { return this._state.contextView.selfSummary } + public get environmentSummary(): string { return this._state.contextView.environmentSummary } // Setters (Partial updates allowed) public update(updates: Partial): void { this._state = { ...this._state, ...updates } } - public updateSelf(updates: Partial): void { - this._state.self = { ...this._state.self, ...updates } - } - - public updateEnvironment(updates: Partial): void { - this._state.environment = { ...this._state.environment, ...updates } + public updateContextView(updates: Partial): void { + this._state.contextView = { ...this._state.contextView, ...updates } } public getSnapshot(): BlackboardState { - // Return a deep copy or safe reference? - // For now, return a shallow copy of the state structure return { ...this._state, - self: { ...this._state.self }, // location (Vec3) is an object, but usually treated efficiently. - environment: { ...this._state.environment, nearbyPlayers: [...this._state.environment.nearbyPlayers], nearbyEntities: [...this._state.environment.nearbyEntities] }, + contextView: { ...this._state.contextView }, } } } diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index eb97c16b0..4cc290406 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -5,12 +5,14 @@ import type { TaskExecutor } from '../action/task-executor' import type { ActionInstruction } from '../action/types' import type { EventManager } from '../perception/event-manager' import type { BotEvent, MineflayerWithAgents, StimulusPayload } from '../types' +import type { ReflexManager } from '../reflex/reflex-manager' import { system, user } from 'neuri/openai' import { config } from '../../composables/config' import { DebugService } from '../../debug-server' import { Blackboard } from './blackboard' +import { buildConsciousContextView } from './context-view' import { generateBrainSystemPrompt } from './prompts/brain-prompt' interface BrainDeps { @@ -18,6 +20,7 @@ interface BrainDeps { neuri: Neuri logger: Logg taskExecutor: TaskExecutor + reflexManager: ReflexManager } interface LLMResponse { @@ -186,19 +189,10 @@ export class Brain { } } - private updatePerception(bot: MineflayerWithAgents): void { - const pos = bot.bot.entity.position - this.blackboard.updateSelf({ - location: pos, - health: bot.bot.health, - food: bot.bot.food, - }) - - this.blackboard.updateEnvironment({ - time: bot.bot.time.isDay ? 'day' : 'night', - weather: bot.bot.isRaining ? 'rain' : 'clear', - nearbyPlayers: Object.keys(bot.bot.players).filter(p => p !== bot.bot.username), - }) + private updatePerception(_bot: MineflayerWithAgents): void { + const ctx = this.deps.reflexManager.getContextSnapshot() + const view = buildConsciousContextView(ctx) + this.blackboard.updateContextView(view) // Sync Blackboard to Debug this.debugService.updateBlackboard(this.blackboard) diff --git a/services/minecraft/src/cognitive/conscious/context-view.ts b/services/minecraft/src/cognitive/conscious/context-view.ts new file mode 100644 index 000000000..2321f24c8 --- /dev/null +++ b/services/minecraft/src/cognitive/conscious/context-view.ts @@ -0,0 +1,19 @@ +import type { ReflexContextState } from '../reflex/context' + +export interface ConsciousContextView { + selfSummary: string + environmentSummary: string +} + +export function buildConsciousContextView(ctx: ReflexContextState): ConsciousContextView { + const selfSummary = `Position ${String(ctx.self.location)} Health ${ctx.self.health}/20 Food ${ctx.self.food}/20 Oxygen ${ctx.self.oxygen}/20 Holding ${ctx.self.holding ?? 'nothing'}` + + const players = ctx.environment.nearbyPlayers.map(p => p.name).join(',') + const entities = ctx.environment.nearbyEntities.map(e => e.name).join(',') + const environmentSummary = `${ctx.environment.time} ${ctx.environment.weather} Nearby players [${players}] Nearby entities [${entities}] Light ${ctx.environment.lightLevel}` + + return { + selfSummary, + environmentSummary, + } +} diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts index 3de4cb566..e94da1a5c 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts @@ -68,7 +68,7 @@ The following blackboard provides you with information about your current state: Goal: "${blackboard.goal}" Thought: "${blackboard.thought}" Strategy: "${blackboard.strategy}" -Self: Position ${blackboard.self.location} Health ${blackboard.self.health}/20 Food ${blackboard.self.food}/20 -Environment: ${blackboard.environment.time} ${blackboard.environment.weather} Nearby entities [${blackboard.environment.nearbyEntities.join(',')}] +Self: ${blackboard.selfSummary} +Environment: ${blackboard.environmentSummary} ` } diff --git a/services/minecraft/src/cognitive/container.ts b/services/minecraft/src/cognitive/container.ts index 7f259d448..719c501c1 100644 --- a/services/minecraft/src/cognitive/container.ts +++ b/services/minecraft/src/cognitive/container.ts @@ -81,7 +81,13 @@ export function createAgentContainer(options: { taskExecutor: asClass(TaskExecutor).singleton(), - brain: asClass(Brain).singleton(), + brain: asClass(Brain) + .singleton() + .inject((c) => { + return { + reflexManager: c.resolve('reflexManager'), + } + }), reflexManager: asClass(ReflexManager).singleton(), }) diff --git a/services/minecraft/src/cognitive/index.ts b/services/minecraft/src/cognitive/index.ts index 603927c3f..e4dc88c33 100644 --- a/services/minecraft/src/cognitive/index.ts +++ b/services/minecraft/src/cognitive/index.ts @@ -44,6 +44,7 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug perceptionPipeline.init(botWithAgents) tickHandler = ({ delta }) => { + reflexManager.tick(delta) perceptionPipeline.tick(delta) } @@ -91,6 +92,9 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug const perceptionPipeline = container.resolve('perceptionPipeline') perceptionPipeline.destroy() + + const reflexManager = container.resolve('reflexManager') + reflexManager.destroy() } if (tickHandler) { diff --git a/services/minecraft/src/cognitive/perception/attention-detector.ts b/services/minecraft/src/cognitive/perception/attention-detector.ts index 696c8e311..139516b20 100644 --- a/services/minecraft/src/cognitive/perception/attention-detector.ts +++ b/services/minecraft/src/cognitive/perception/attention-detector.ts @@ -1,6 +1,6 @@ import type { Logg } from '@guiiai/logg' -import type { RawPerceptionEvent } from './raw-events' +import type { RawPerceptionEvent } from './types/raw-events' import { LeakyBucket } from './leaky-bucket' diff --git a/services/minecraft/src/cognitive/perception/frame.ts b/services/minecraft/src/cognitive/perception/frame.ts index 93d87c3b2..9fed4637f 100644 --- a/services/minecraft/src/cognitive/perception/frame.ts +++ b/services/minecraft/src/cognitive/perception/frame.ts @@ -1,4 +1,4 @@ -import type { RawPerceptionEvent } from './raw-events' +import type { RawPerceptionEvent } from './types/raw-events' export type PerceptionFrameSource = 'minecraft' diff --git a/services/minecraft/src/cognitive/perception/mineflayer-perception-collector.ts b/services/minecraft/src/cognitive/perception/mineflayer-perception-collector.ts index 0e5cfa6c7..a750861b8 100644 --- a/services/minecraft/src/cognitive/perception/mineflayer-perception-collector.ts +++ b/services/minecraft/src/cognitive/perception/mineflayer-perception-collector.ts @@ -10,7 +10,7 @@ import type { SightedArmSwingEvent, SightedEntityMovedEvent, SightedSneakToggleEvent, -} from './raw-events' +} from './types/raw-events' export class MineflayerPerceptionCollector { private bot: MineflayerWithAgents | null = null diff --git a/services/minecraft/src/cognitive/perception/normalizer-stage.ts b/services/minecraft/src/cognitive/perception/normalizer-stage.ts index e6f81c97c..22c39161c 100644 --- a/services/minecraft/src/cognitive/perception/normalizer-stage.ts +++ b/services/minecraft/src/cognitive/perception/normalizer-stage.ts @@ -1,6 +1,6 @@ import type { PerceptionFrame } from './frame' -import type { RawPerceptionEvent } from './raw-events' -import type { PerceptionStage } from './stage' +import type { RawPerceptionEvent } from './types/raw-events' +import type { PerceptionStage } from './types/stage' function getDistance(raw: RawPerceptionEvent): number | undefined { return (raw as any).distance diff --git a/services/minecraft/src/cognitive/perception/pipeline.ts b/services/minecraft/src/cognitive/perception/pipeline.ts index 53e0dd1fe..999008aa2 100644 --- a/services/minecraft/src/cognitive/perception/pipeline.ts +++ b/services/minecraft/src/cognitive/perception/pipeline.ts @@ -12,7 +12,7 @@ import { createPerceptionFrameFromRawEvent } from './frame' import { MineflayerPerceptionCollector } from './mineflayer-perception-collector' import { NormalizerStage } from './normalizer-stage' import { RawEventBuffer } from './raw-event-buffer' -import type { PerceptionStage } from './stage' +import type { PerceptionStage } from './types/stage' export class PerceptionPipeline { private readonly buffer = new RawEventBuffer() diff --git a/services/minecraft/src/cognitive/perception/raw-events.ts b/services/minecraft/src/cognitive/perception/raw-events.ts index 70ddd4c18..8efb34698 100644 --- a/services/minecraft/src/cognitive/perception/raw-events.ts +++ b/services/minecraft/src/cognitive/perception/raw-events.ts @@ -1,74 +1 @@ -import type { Vec3 } from 'vec3' - -export type PerceptionModality = 'sighted' | 'heard' | 'felt' - -export interface RawPerceptionEventBase { - modality: PerceptionModality - timestamp: number - source: 'minecraft' - pos?: Vec3 -} - -export interface SightedEntityMovedEvent extends RawPerceptionEventBase { - modality: 'sighted' - kind: 'entity_moved' - entityType: 'player' | 'mob' - entityId: string - displayName?: string - distance: number - hasLineOfSight: boolean -} - -export interface SightedArmSwingEvent extends RawPerceptionEventBase { - modality: 'sighted' - kind: 'arm_swing' - entityType: 'player' - entityId: string - displayName?: string - distance: number - hasLineOfSight: boolean -} - -export interface SightedSneakToggleEvent extends RawPerceptionEventBase { - modality: 'sighted' - kind: 'sneak_toggle' - entityType: 'player' - entityId: string - displayName?: string - distance: number - hasLineOfSight: boolean - sneaking: boolean -} - -export type SightedEvent = SightedEntityMovedEvent | SightedArmSwingEvent | SightedSneakToggleEvent - -export interface HeardSoundEvent extends RawPerceptionEventBase { - modality: 'heard' - kind: 'sound' - soundId: string - distance: number - inferredEntityType?: 'player' | 'mob' - inferredEntityId?: string -} - -export type HeardEvent = HeardSoundEvent - -export interface FeltDamageTakenEvent extends RawPerceptionEventBase { - modality: 'felt' - kind: 'damage_taken' - amount?: number - attackerEntityType?: 'player' | 'mob' - attackerEntityId?: string - distance?: number -} - -export interface FeltItemCollectedEvent extends RawPerceptionEventBase { - modality: 'felt' - kind: 'item_collected' - itemName: string - count?: number -} - -export type FeltEvent = FeltDamageTakenEvent | FeltItemCollectedEvent - -export type RawPerceptionEvent = SightedEvent | HeardEvent | FeltEvent +export * from './types/raw-events' diff --git a/services/minecraft/src/cognitive/perception/stage.ts b/services/minecraft/src/cognitive/perception/stage.ts index 6e6a895d3..2fc90ef72 100644 --- a/services/minecraft/src/cognitive/perception/stage.ts +++ b/services/minecraft/src/cognitive/perception/stage.ts @@ -1,7 +1 @@ -import type { PerceptionFrame } from './frame' - -export interface PerceptionStage { - name: string - tick?: (deltaMs: number) => void - handle: (frame: PerceptionFrame) => PerceptionFrame | null -} +export * from './types/stage' diff --git a/services/minecraft/src/cognitive/perception/types/raw-events.ts b/services/minecraft/src/cognitive/perception/types/raw-events.ts new file mode 100644 index 000000000..ed18c6a86 --- /dev/null +++ b/services/minecraft/src/cognitive/perception/types/raw-events.ts @@ -0,0 +1,74 @@ +import type { Vec3 } from 'vec3' + +export type PerceptionModality = 'sighted' | 'heard' | 'felt' + +export interface RawPerceptionEventBase { + modality: PerceptionModality + timestamp: number + source: 'minecraft' + pos?: Vec3 +} + +export interface SightedEntityMovedEvent extends RawPerceptionEventBase { + modality: 'sighted' + kind: 'entity_moved' + entityType: 'player' | 'mob' + entityId: string + displayName?: string + distance: number + hasLineOfSight: boolean +} + +export interface SightedArmSwingEvent extends RawPerceptionEventBase { + modality: 'sighted' + kind: 'arm_swing' + entityType: 'player' + entityId: string + displayName?: string + distance: number + hasLineOfSight: boolean +} + +export interface SightedSneakToggleEvent extends RawPerceptionEventBase { + modality: 'sighted' + kind: 'sneak_toggle' + entityType: 'player' + entityId: string + displayName?: string + distance: number + hasLineOfSight: boolean + sneaking: boolean +} + +export type SightedEvent = SightedEntityMovedEvent | SightedArmSwingEvent | SightedSneakToggleEvent + +export interface HeardSoundEvent extends RawPerceptionEventBase { + modality: 'heard' + kind: 'sound' + soundId: string + distance: number + inferredEntityType?: 'player' | 'mob' + inferredEntityId?: string +} + +export type HeardEvent = HeardSoundEvent + +export interface FeltDamageTakenEvent extends RawPerceptionEventBase { + modality: 'felt' + kind: 'damage_taken' + amount?: number + attackerEntityType?: 'player' | 'mob' + attackerEntityId?: string + distance?: number +} + +export interface FeltItemCollectedEvent extends RawPerceptionEventBase { + modality: 'felt' + kind: 'item_collected' + itemName: string + count?: number +} + +export type FeltEvent = FeltDamageTakenEvent | FeltItemCollectedEvent + +export type RawPerceptionEvent = SightedEvent | HeardEvent | FeltEvent diff --git a/services/minecraft/src/cognitive/perception/types/stage.ts b/services/minecraft/src/cognitive/perception/types/stage.ts new file mode 100644 index 000000000..0c1de6d40 --- /dev/null +++ b/services/minecraft/src/cognitive/perception/types/stage.ts @@ -0,0 +1,7 @@ +import type { PerceptionFrame } from '../frame' + +export interface PerceptionStage { + name: string + tick?: (deltaMs: number) => void + handle: (frame: PerceptionFrame) => PerceptionFrame | null +} diff --git a/services/minecraft/src/cognitive/reflex/behavior.ts b/services/minecraft/src/cognitive/reflex/behavior.ts new file mode 100644 index 000000000..f622a4738 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/behavior.ts @@ -0,0 +1 @@ +export * from './types/behavior' diff --git a/services/minecraft/src/cognitive/reflex/behaviors/greeting.ts b/services/minecraft/src/cognitive/reflex/behaviors/greeting.ts new file mode 100644 index 000000000..593511469 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/behaviors/greeting.ts @@ -0,0 +1,40 @@ +import type { ReflexBehavior } from '../types/behavior' + +export const greetingBehavior: ReflexBehavior = { + id: 'greeting', + modes: ['social'], + cooldownMs: 10_000, + when: (ctx) => { + const msg = ctx.social.lastMessage + if (!msg) + return false + + const lower = msg.toLowerCase().trim() + return lower === 'hi' || lower === 'hello' + }, + score: (ctx) => { + if (!ctx.social.lastSpeaker) + return 0 + + const lastGreetAt = ctx.social.lastGreetingAtBySpeaker[ctx.social.lastSpeaker] + if (lastGreetAt && ctx.now - lastGreetAt < 10_000) + return 0 + + return 10 + }, + run: ({ bot, context }) => { + const snap = context.getSnapshot() + const speaker = snap.social.lastSpeaker + if (!speaker) + return + + bot.bot.chat('Hi there! (Reflex)') + + context.updateSocial({ + lastGreetingAtBySpeaker: { + ...snap.social.lastGreetingAtBySpeaker, + [speaker]: snap.now, + }, + }) + }, +} diff --git a/services/minecraft/src/cognitive/reflex/context.ts b/services/minecraft/src/cognitive/reflex/context.ts new file mode 100644 index 000000000..79dde84c1 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/context.ts @@ -0,0 +1,110 @@ +import type { Vec3 } from 'vec3' + +export interface ReflexSelfState { + location: Vec3 | null + holding: string | null + health: number + food: number + oxygen: number +} + +export interface ReflexEnvironmentState { + time: 'day' | 'night' | 'sunset' | 'sunrise' + weather: 'clear' | 'rain' | 'thunder' + nearbyPlayers: Array<{ name: string, distance?: number }> + nearbyEntities: Array<{ name: string, distance?: number, kind?: string }> + lightLevel: number +} + +export interface ReflexSocialState { + lastSpeaker: string | null + lastMessage: string | null + lastMessageAt: number | null + lastGreetingAtBySpeaker: Record +} + +export interface ReflexThreatState { + threatScore: number + lastThreatAt: number | null + lastThreatSource: string | null +} + +export interface ReflexContextState { + now: number + self: ReflexSelfState + environment: ReflexEnvironmentState + social: ReflexSocialState + threat: ReflexThreatState +} + +export class ReflexContext { + private state: ReflexContextState + + constructor() { + this.state = { + now: Date.now(), + self: { + location: null, + holding: null, + health: 20, + food: 20, + oxygen: 20, + }, + environment: { + time: 'day', + weather: 'clear', + nearbyPlayers: [], + nearbyEntities: [], + lightLevel: 15, + }, + social: { + lastSpeaker: null, + lastMessage: null, + lastMessageAt: null, + lastGreetingAtBySpeaker: {}, + }, + threat: { + threatScore: 0, + lastThreatAt: null, + lastThreatSource: null, + }, + } + } + + public getSnapshot(): ReflexContextState { + return { + ...this.state, + self: { ...this.state.self }, + environment: { + ...this.state.environment, + nearbyPlayers: this.state.environment.nearbyPlayers.map(p => ({ ...p })), + nearbyEntities: this.state.environment.nearbyEntities.map(e => ({ ...e })), + }, + social: { + ...this.state.social, + lastGreetingAtBySpeaker: { ...this.state.social.lastGreetingAtBySpeaker }, + }, + threat: { ...this.state.threat }, + } + } + + public updateNow(now: number): void { + this.state.now = now + } + + public updateSelf(patch: Partial): void { + this.state.self = { ...this.state.self, ...patch } + } + + public updateEnvironment(patch: Partial): void { + this.state.environment = { ...this.state.environment, ...patch } + } + + public updateSocial(patch: Partial): void { + this.state.social = { ...this.state.social, ...patch } + } + + public updateThreat(patch: Partial): void { + this.state.threat = { ...this.state.threat, ...patch } + } +} diff --git a/services/minecraft/src/cognitive/reflex/modes.ts b/services/minecraft/src/cognitive/reflex/modes.ts new file mode 100644 index 000000000..1a42bc514 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/modes.ts @@ -0,0 +1,13 @@ +import type { ReflexContextState } from './context' + +export type ReflexModeId = 'idle' | 'social' | 'alert' + +export function selectMode(ctx: ReflexContextState): ReflexModeId { + if (ctx.threat.threatScore > 0) + return 'alert' + + if (ctx.social.lastMessageAt && ctx.now - ctx.social.lastMessageAt < 15_000) + return 'social' + + return 'idle' +} diff --git a/services/minecraft/src/cognitive/reflex/reflex-manager.test.ts b/services/minecraft/src/cognitive/reflex/reflex-manager.test.ts new file mode 100644 index 000000000..19f7ce9e2 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/reflex-manager.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from 'vitest' + +import { EventManager } from '../perception/event-manager' + +import { ReflexManager } from './reflex-manager' + +function makeLogger() { + return { + log: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + withFields: vi.fn(() => makeLogger()), + withError: vi.fn(() => makeLogger()), + } as any +} + +function makeBot() { + const bot = { + bot: { + username: 'bot', + chat: vi.fn(), + entity: { + position: { x: 0, y: 0, z: 0 }, + }, + health: 20, + food: 20, + oxygenLevel: 20, + heldItem: null, + time: { isDay: true }, + isRaining: false, + players: {}, + }, + } + + return bot as any +} + +describe('ReflexManager', () => { + it('handles greeting via reflex and marks stimulus event handled', () => { + const eventManager = new EventManager() + const logger = makeLogger() + const reflex = new ReflexManager({ eventManager, logger }) + + const bot = makeBot() + reflex.init(bot) + + const stimulus: any = { + type: 'stimulus', + payload: { content: 'hello' }, + source: { type: 'minecraft', id: 'alice' }, + timestamp: Date.now(), + } + + eventManager.emit(stimulus) + + expect(stimulus.handled).toBe(true) + expect(bot.bot.chat).toHaveBeenCalled() + + reflex.destroy() + }) +}) diff --git a/services/minecraft/src/cognitive/reflex/reflex-manager.ts b/services/minecraft/src/cognitive/reflex/reflex-manager.ts index 6b46a0ed0..c14bc3b47 100644 --- a/services/minecraft/src/cognitive/reflex/reflex-manager.ts +++ b/services/minecraft/src/cognitive/reflex/reflex-manager.ts @@ -3,38 +3,69 @@ import type { Logg } from '@guiiai/logg' import type { EventManager } from '../perception/event-manager' import type { BotEvent, MineflayerWithAgents, StimulusPayload } from '../types' +import type { ReflexContextState } from './context' + +import { greetingBehavior } from './behaviors/greeting' +import { ReflexRuntime } from './runtime' + export class ReflexManager { + private bot: MineflayerWithAgents | null = null + private readonly runtime: ReflexRuntime + + private readonly onStimulusHandler = (event: BotEvent) => { + this.onStimulus(event) + } + constructor( private readonly deps: { eventManager: EventManager logger: Logg }, - ) {} - - public init(bot: MineflayerWithAgents): void { - // Listen to stimuli as a "subconscious" filter - this.deps.eventManager.on('stimulus', (event) => { - this.onStimulus(bot, event) + ) { + this.runtime = new ReflexRuntime({ + logger: this.deps.logger, }) - // TODO: Listen to world_update for physical reflexes (dodge, flee) + this.runtime.registerBehavior(greetingBehavior) } - private onStimulus(bot: MineflayerWithAgents, event: BotEvent): void { - const { content } = event.payload - const lowerContent = content.toLowerCase().trim() + public init(bot: MineflayerWithAgents): void { + this.bot = bot + this.deps.eventManager.on('stimulus', this.onStimulusHandler) + } - if (lowerContent === 'hi' || lowerContent === 'hello') { - this.deps.logger.log('Reflex: Handling greeting') + public destroy(): void { + this.deps.eventManager.off('stimulus', this.onStimulusHandler) + this.bot = null + } - const reply = 'Hi there! (Reflex)' - if (event.source.reply) { - event.source.reply(reply) - } - else { - bot.bot.chat(reply) - } + public tick(deltaMs: number): void { + if (!this.bot) + return + this.runtime.tick(this.bot, deltaMs) + } + + public getContextSnapshot(): ReflexContextState { + return this.runtime.getContext().getSnapshot() + } + + private onStimulus(event: BotEvent): void { + const bot = this.bot + if (!bot) + return + + const now = Date.now() + + this.runtime.getContext().updateNow(now) + this.runtime.getContext().updateSocial({ + lastSpeaker: event.source.id, + lastMessage: event.payload.content, + lastMessageAt: now, + }) + + const behaviorId = this.runtime.tick(bot, 0) + if (behaviorId) { event.handled = true } } diff --git a/services/minecraft/src/cognitive/reflex/runtime.ts b/services/minecraft/src/cognitive/reflex/runtime.ts new file mode 100644 index 000000000..1e48946c2 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/runtime.ts @@ -0,0 +1,129 @@ +import type { Logg } from '@guiiai/logg' + +import type { MineflayerWithAgents } from '../types' + +import { ReflexContext } from './context' +import type { ReflexBehavior } from './types/behavior' +import { selectMode, type ReflexModeId } from './modes' + +export class ReflexRuntime { + private readonly context = new ReflexContext() + private readonly behaviors: ReflexBehavior[] = [] + private readonly runHistory = new Map() + + private mode: ReflexModeId = 'idle' + private activeBehaviorId: string | null = null + private activeBehaviorUntil: number | null = null + + public constructor( + private readonly deps: { + logger: Logg + }, + ) { } + + public getContext(): ReflexContext { + return this.context + } + + public getMode(): ReflexModeId { + return this.mode + } + + public getActiveBehaviorId(): string | null { + return this.activeBehaviorId + } + + public registerBehavior(behavior: ReflexBehavior): void { + this.behaviors.push(behavior) + } + + public tick(bot: MineflayerWithAgents, deltaMs: number): string | null { + const now = Date.now() + + this.context.updateNow(now) + + // TODO: future refactor: update ReflexContext via world_update/self_update events instead of polling Mineflayer state. + this.context.updateSelf({ + location: bot.bot.entity.position, + health: bot.bot.health, + food: bot.bot.food, + oxygen: bot.bot.oxygenLevel, + holding: bot.bot.heldItem?.name ?? null, + }) + + this.context.updateEnvironment({ + time: bot.bot.time.isDay ? 'day' : 'night', + weather: bot.bot.isRaining ? 'rain' : 'clear', + nearbyPlayers: Object.keys(bot.bot.players) + .filter(p => p !== bot.bot.username) + .map(name => ({ name })), + }) + + this.mode = selectMode(this.context.getSnapshot()) + + if (this.activeBehaviorUntil && now < this.activeBehaviorUntil) + return null + + this.activeBehaviorId = null + this.activeBehaviorUntil = null + + const ctx = this.context.getSnapshot() + + let best: { behavior: ReflexBehavior, score: number } | null = null + for (const behavior of this.behaviors) { + if (!behavior.modes.includes(this.mode)) + continue + + if (!behavior.when(ctx)) + continue + + const score = behavior.score(ctx) + if (score <= 0) + continue + + const history = this.runHistory.get(behavior.id) + const cooldownMs = behavior.cooldownMs ?? 0 + if (history && cooldownMs > 0 && now - history.lastRunAt < cooldownMs) + continue + + if (!best || score > best.score) + best = { behavior, score } + } + + if (!best) + return null + + this.activeBehaviorId = best.behavior.id + this.runHistory.set(best.behavior.id, { lastRunAt: now }) + + try { + const maybePromise = best.behavior.run({ bot, context: this.context }) + if (maybePromise && typeof (maybePromise as any).then === 'function') { + this.activeBehaviorUntil = now + Math.max(deltaMs, 50) + void (maybePromise as Promise).finally(() => { + // Behavior ends naturally; next tick can run a new one. + this.activeBehaviorUntil = null + this.activeBehaviorId = null + }) + } + else { + // Synchronous behavior ends immediately. + this.activeBehaviorId = null + } + + this.deps.logger.withFields({ + mode: this.mode, + behavior: best.behavior.id, + score: best.score, + }).log('ReflexRuntime: selected') + + return best.behavior.id + } + catch (err) { + this.deps.logger.withError(err as Error).error('ReflexRuntime: behavior failed') + this.activeBehaviorId = null + this.activeBehaviorUntil = null + return null + } + } +} diff --git a/services/minecraft/src/cognitive/reflex/types/behavior.ts b/services/minecraft/src/cognitive/reflex/types/behavior.ts new file mode 100644 index 000000000..5ef0bd2a6 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/types/behavior.ts @@ -0,0 +1,22 @@ +import type { MineflayerWithAgents } from '../../types' + +import type { ReflexContext } from '../context' +import type { ReflexModeId } from '../modes' + +export interface ReflexApi { + bot: MineflayerWithAgents + context: ReflexContext +} + +export interface ReflexBehavior { + id: string + modes: ReflexModeId[] + cooldownMs?: number + when: (ctx: ReturnType) => boolean + score: (ctx: ReturnType) => number + run: (api: ReflexApi) => Promise | void +} + +export interface BehaviorRunRecord { + lastRunAt: number +} diff --git a/services/minecraft/src/cognitive/types.ts b/services/minecraft/src/cognitive/types.ts index 8bf373c64..679c9fc91 100644 --- a/services/minecraft/src/cognitive/types.ts +++ b/services/minecraft/src/cognitive/types.ts @@ -1,61 +1 @@ -import type { Client } from '@proj-airi/server-sdk' -import type { Neuri } from 'neuri' - -import type { Mineflayer } from '../libs/mineflayer' -import type { ActionAgent, ChatAgent, PlanningAgent } from '../libs/mineflayer/base-agent' - -export interface LLMConfig { - agent: Neuri - model?: string - retryLimit?: number - delayInterval?: number - maxContextLength?: number -} - -export interface LLMResponse { - content: string - usage?: any -} - -export interface MineflayerWithAgents extends Mineflayer { - planning: PlanningAgent - action: ActionAgent - chat: ChatAgent -} - -export interface CognitiveEngineOptions { - agent: Neuri - airiClient: Client -} - -// TODO: currently stimulus is just chat events, consider renaming to 'input' or 'user_interaction' -export type EventCategory = 'stimulus' | 'perception' | 'feedback' | 'world_update' | 'system_alert' - -export interface BotEventSource { - type: 'minecraft' | 'airi' | 'system' - id: string // Agent/Source identifier - reply?: (message: string) => void -} - -export interface BotEvent { - type: EventCategory - payload: T - source: BotEventSource - timestamp: number - // Layered Architecture Metadata - priority?: number // Higher is more urgent - handled?: boolean // Set by Reflex layer to inhibit Conscious layer -} - -export interface StimulusPayload { - content: string - metadata?: { - entity?: any // prismarine-entity Entity - displayName?: string - } -} - -export interface WorldUpdatePayload { - event: string - data: any -} +export * from './types/index' diff --git a/services/minecraft/src/cognitive/types/index.ts b/services/minecraft/src/cognitive/types/index.ts new file mode 100644 index 000000000..9af6a9919 --- /dev/null +++ b/services/minecraft/src/cognitive/types/index.ts @@ -0,0 +1,61 @@ +import type { Client } from '@proj-airi/server-sdk' +import type { Neuri } from 'neuri' + +import type { Mineflayer } from '../../libs/mineflayer' +import type { ActionAgent, ChatAgent, PlanningAgent } from '../../libs/mineflayer/base-agent' + +export interface LLMConfig { + agent: Neuri + model?: string + retryLimit?: number + delayInterval?: number + maxContextLength?: number +} + +export interface LLMResponse { + content: string + usage?: any +} + +export interface MineflayerWithAgents extends Mineflayer { + planning: PlanningAgent + action: ActionAgent + chat: ChatAgent +} + +export interface CognitiveEngineOptions { + agent: Neuri + airiClient: Client +} + +// TODO: currently stimulus is just chat events, consider renaming to 'input' or 'user_interaction' +export type EventCategory = 'stimulus' | 'perception' | 'feedback' | 'world_update' | 'system_alert' + +export interface BotEventSource { + type: 'minecraft' | 'airi' | 'system' + id: string // Agent/Source identifier + reply?: (message: string) => void +} + +export interface BotEvent { + type: EventCategory + payload: T + source: BotEventSource + timestamp: number + // Layered Architecture Metadata + priority?: number // Higher is more urgent + handled?: boolean // Set by Reflex layer to inhibit Conscious layer +} + +export interface StimulusPayload { + content: string + metadata?: { + entity?: any // prismarine-entity Entity + displayName?: string + } +} + +export interface WorldUpdatePayload { + event: string + data: any +}