diff --git a/services/minecraft/src/cognitive/conscious/context-view.ts b/services/minecraft/src/cognitive/conscious/context-view.ts index ae864d566..f15b9a544 100644 --- a/services/minecraft/src/cognitive/conscious/context-view.ts +++ b/services/minecraft/src/cognitive/conscious/context-view.ts @@ -12,7 +12,23 @@ export function buildConsciousContextView(ctx: ReflexContextState): ConsciousCon const players = ctx.environment.nearbyPlayers.map(p => p.name).join(',') const entities = ctx.environment.nearbyEntities.map(e => e.name).join(',') - const environmentSummary = `${ctx.environment.time} ${ctx.environment.weather} Nearby players [${players}] Nearby entities [${entities}] Light ${ctx.environment.lightLevel}` + + const gaze = ctx.environment.nearbyPlayersGaze + .map((g) => { + const hit = g.hitBlock + ? `${g.hitBlock.name}@(${Math.round(g.hitBlock.pos.x)}, ${Math.round(g.hitBlock.pos.y)}, ${Math.round(g.hitBlock.pos.z)})` + : 'air' + + if (g.hitBlock) + return `${g.name}->${hit}` + + const lp = `(${Math.round(g.lookPoint.x)}, ${Math.round(g.lookPoint.y)}, ${Math.round(g.lookPoint.z)})` + return `${g.name}->${hit} lookPoint${lp}` + }) + .join(' | ') + const gazeSummary = ctx.environment.nearbyPlayersGaze.length > 0 ? ` Nearby player gaze [${gaze}]` : '' + + const environmentSummary = `${ctx.environment.time} ${ctx.environment.weather} Nearby players [${players}] Nearby entities [${entities}] Light ${ctx.environment.lightLevel}${gazeSummary}` return { selfSummary, diff --git a/services/minecraft/src/cognitive/index.ts b/services/minecraft/src/cognitive/index.ts index 6e10988a1..0d3ba8641 100644 --- a/services/minecraft/src/cognitive/index.ts +++ b/services/minecraft/src/cognitive/index.ts @@ -6,6 +6,7 @@ import { DebugService } from '../debug' import { ChatMessageHandler } from '../libs/mineflayer' import { createAgentContainer } from './container' import { createPerceptionFrameFromChat } from './perception/frame' +import { computeNearbyPlayerGaze } from './reflex/gaze' export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlugin { let container: ReturnType @@ -52,6 +53,23 @@ export function CognitiveEngine(options: CognitiveEngineOptions): MineflayerPlug // Initialize perception pipeline (raw events + detectors) perceptionPipeline.init(botWithAgents) + let tickCount = 0 + bot.onTick('tick', () => { + tickCount++ + if (tickCount % 5 !== 0) + return + + const gaze = computeNearbyPlayerGaze(bot.bot, { maxDistance: 32, nearbyDistance: 16 }) + reflexManager.updateEnvironment({ + nearbyPlayersGaze: gaze.map(g => ({ + name: g.playerName, + distanceToSelf: g.distanceToSelf, + lookPoint: g.lookPoint, + hitBlock: g.hitBlock, + })), + }) + }) + // Resolve EventBus and subscribe to forward events to debug timeline const eventBus = container.resolve('eventBus') eventBus.subscribe('*', (event) => { diff --git a/services/minecraft/src/cognitive/reflex/context.ts b/services/minecraft/src/cognitive/reflex/context.ts index e36178ad2..b0c44f774 100644 --- a/services/minecraft/src/cognitive/reflex/context.ts +++ b/services/minecraft/src/cognitive/reflex/context.ts @@ -11,6 +11,12 @@ export interface ReflexEnvironmentState { time: 'day' | 'night' | 'sunset' | 'sunrise' weather: 'clear' | 'rain' | 'thunder' nearbyPlayers: Array<{ name: string, distance?: number }> + nearbyPlayersGaze: Array<{ + name: string + distanceToSelf: number + lookPoint: { x: number, y: number, z: number } + hitBlock: null | { name: string, pos: { x: number, y: number, z: number } } + }> nearbyEntities: Array<{ name: string, distance?: number, kind?: string }> lightLevel: number } @@ -61,6 +67,7 @@ export class ReflexContext { time: 'day', weather: 'clear', nearbyPlayers: [], + nearbyPlayersGaze: [], nearbyEntities: [], lightLevel: 15, }, @@ -92,6 +99,16 @@ export class ReflexContext { environment: { ...this.state.environment, nearbyPlayers: this.state.environment.nearbyPlayers.map(p => ({ ...p })), + nearbyPlayersGaze: this.state.environment.nearbyPlayersGaze.map(p => ({ + ...p, + lookPoint: { ...p.lookPoint }, + hitBlock: p.hitBlock + ? { + ...p.hitBlock, + pos: { ...p.hitBlock.pos }, + } + : null, + })), nearbyEntities: this.state.environment.nearbyEntities.map(e => ({ ...e })), }, social: { diff --git a/services/minecraft/src/cognitive/reflex/gaze.ts b/services/minecraft/src/cognitive/reflex/gaze.ts new file mode 100644 index 000000000..48e77bb80 --- /dev/null +++ b/services/minecraft/src/cognitive/reflex/gaze.ts @@ -0,0 +1,141 @@ +import type { Bot } from 'mineflayer' + +import { Vec3 } from 'vec3' + +interface Vec3Like { x: number, y: number, z: number } + +interface PlayerEntityLike { + type?: string + username?: string + position: Vec3 + yaw?: number + pitch?: number +} + +export interface PlayerGazeResult { + playerName: string + distanceToSelf: number + lookPoint: Vec3Like + hitBlock: null | { + name: string + pos: Vec3Like + } +} + +function directionFromYawPitch(yaw: number, pitch: number): Vec3Like { + const x = -Math.sin(yaw) * Math.cos(pitch) + const y = Math.sin(pitch) + const z = -Math.cos(yaw) * Math.cos(pitch) + return { x, y, z } +} + +function normalize(v: Vec3Like): Vec3Like { + const len = Math.sqrt(v.x * v.x + v.y * v.y + v.z * v.z) || 1 + return { x: v.x / len, y: v.y / len, z: v.z / len } +} + +function add(a: Vec3Like, b: Vec3Like): Vec3Like { + return { x: a.x + b.x, y: a.y + b.y, z: a.z + b.z } +} + +function scale(v: Vec3Like, s: number): Vec3Like { + return { x: v.x * s, y: v.y * s, z: v.z * s } +} + +function floorVec(v: Vec3Like): Vec3Like { + return { x: Math.floor(v.x), y: Math.floor(v.y), z: Math.floor(v.z) } +} + +function distance(a: Vec3Like, b: Vec3Like): number { + const dx = a.x - b.x + const dy = a.y - b.y + const dz = a.z - b.z + return Math.sqrt(dx * dx + dy * dy + dz * dz) +} + +export function rayTraceBlockFromEntity( + bot: Bot, + entity: PlayerEntityLike, + options?: { + maxDistance?: number + step?: number + eyeHeight?: number + }, +): { lookPoint: Vec3Like, hitBlock: PlayerGazeResult['hitBlock'] } { + const maxDistance = options?.maxDistance ?? 32 + const step = options?.step ?? 0.25 + const eyeHeight = options?.eyeHeight ?? 1.62 + + const yaw = entity.yaw ?? 0 + const pitch = entity.pitch ?? 0 + + const dir = normalize(directionFromYawPitch(yaw, pitch)) + const origin = add(entity.position, { x: 0, y: eyeHeight, z: 0 }) + + const lookPoint = add(origin, scale(dir, maxDistance)) + + let lastBlockPosKey: string | null = null + + for (let d = 0; d <= maxDistance; d += step) { + const p = add(origin, scale(dir, d)) + const bp = floorVec(p) + const key = `${bp.x},${bp.y},${bp.z}` + if (key === lastBlockPosKey) + continue + lastBlockPosKey = key + + const block = bot.blockAt(new Vec3(bp.x, bp.y, bp.z)) + if (!block) + continue + + if (block.name !== 'air') { + return { + lookPoint, + hitBlock: { + name: block.name, + pos: { x: block.position.x, y: block.position.y, z: block.position.z }, + }, + } + } + } + + return { lookPoint, hitBlock: null } +} + +export function computeNearbyPlayerGaze( + bot: Bot, + options?: { + maxDistance?: number + nearbyDistance?: number + }, +): PlayerGazeResult[] { + const self = bot.entity + if (!self) + return [] + + const nearbyDistance = options?.nearbyDistance ?? 16 + + const players = Object.values(bot.players ?? {}) + .map(p => p?.entity as PlayerEntityLike | undefined) + .filter((e): e is PlayerEntityLike => Boolean(e && e.type === 'player' && e.username)) + .filter(e => e.username !== bot.username) + + const selfPos = self.position + + return players + .map((p) => { + const dist = distance(selfPos, p.position) + return { p, dist } + }) + .filter(x => x.dist <= nearbyDistance) + .sort((a, b) => a.dist - b.dist) + .map(({ p, dist }) => { + const { lookPoint, hitBlock } = rayTraceBlockFromEntity(bot, p, { maxDistance: options?.maxDistance ?? 32 }) + return { + playerName: p.username!, + distanceToSelf: dist, + lookPoint, + hitBlock, + } + }) +} diff --git a/services/minecraft/src/cognitive/reflex/reflex-manager.ts b/services/minecraft/src/cognitive/reflex/reflex-manager.ts index ef6466b30..4bd5e20d3 100644 --- a/services/minecraft/src/cognitive/reflex/reflex-manager.ts +++ b/services/minecraft/src/cognitive/reflex/reflex-manager.ts @@ -53,6 +53,10 @@ export class ReflexManager { return this.runtime.getContext().getSnapshot() } + public updateEnvironment(patch: Partial): void { + this.runtime.getContext().updateEnvironment(patch) + } + private onSignal(event: TracedEvent): void { const bot = this.bot if (!bot)