refactor(minecraft): replace blocking followPlayer with reflex auto-follow system

Replace async followPlayer skill with sync reflex-based auto-follow that runs during idle mode, add clearFollowTarget action to disable auto-follow before independent movement, add autonomy state to ReflexContext tracking followPlayer/followDistance/followActive/followLastError, implement reconcileAutoFollow in ReflexRuntime using pathfinder GoalFollow that pauses during work/wander/alert modes, add followControl:

services/minecraft/src/cognitive/action/llm-actions.ts

@@ -154,21 +154,42 @@ export const actionsList: Action[] = [
   },
   {
     name: 'followPlayer',
-    description: 'Endlessly follow the given player.',
-    execution: 'async',
+    description: 'Set idle auto-follow target handled by reflex runtime. While idle, the bot will keep following this player until cleared.',
+    execution: 'sync',
+    readonly: true,
     schema: z.object({
       player_name: z.string().describe('name of the player to follow.'),
       follow_dist: z.number().describe('The distance to follow from.').min(0),
     }),
-    perform: mineflayer => async (player_name: string, follow_dist: number) => {
-      await skills.followPlayer(mineflayer, player_name, follow_dist)
-      return `Following player [${player_name}]`
+    perform: mineflayer => (player_name: string, follow_dist: number) => {
+      const reflexManager = (mineflayer as any).reflexManager
+      if (!reflexManager || typeof reflexManager.setFollowTarget !== 'function')
+        throw new Error('Reflex follow manager is unavailable')
+
+      reflexManager.setFollowTarget(player_name, follow_dist)
+      return `Auto-follow enabled for player [${player_name}] at distance ${follow_dist}`
+    },
+  },
+  {
+    name: 'clearFollowTarget',
+    description: 'Disable idle auto-follow. Use this before independent exploration or when you no longer want to shadow a player.',
+    execution: 'sync',
+    readonly: true,
+    schema: z.object({}),
+    perform: mineflayer => () => {
+      const reflexManager = (mineflayer as any).reflexManager
+      if (!reflexManager || typeof reflexManager.clearFollowTarget !== 'function')
+        throw new Error('Reflex follow manager is unavailable')
+
+      reflexManager.clearFollowTarget()
+      return 'Auto-follow disabled'
     },
   },
   {
     name: 'goToCoordinate',
     description: 'Go to the given x, y, z location.',
     execution: 'async',
+    followControl: 'detach',
     schema: z.object({
       x: z.number().describe('The x coordinate.'),
       y: z.number().describe('The y coordinate.').min(-64).max(320),

services/minecraft/src/cognitive/conscious/brain.ts

@@ -267,6 +267,9 @@ export class Brain {
           }
 
           const actionDef = actionDefs.get(action.tool)
+          if (actionDef?.followControl === 'detach')
+            this.deps.reflexManager.clearFollowTarget()
+
           const isPhysicalAction = action.tool !== 'skip' && !actionDef?.readonly
 
           if (isPhysicalAction) {
@@ -344,7 +347,7 @@ export class Brain {
       parts.push(`[STATE] giveUp active (${remainingSec}s left). reason=${this.giveUpReason ?? 'unknown'}`)
     }
 
-    parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, event, now, mem, lastRun, lastAction.')
+    parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, mem, lastRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.')
 
     return parts.join('\n\n')
   }

services/minecraft/src/cognitive/conscious/context-view.ts

@@ -8,27 +8,17 @@ export interface ConsciousContextView {
 export function buildConsciousContextView(ctx: ReflexContextState): ConsciousContextView {
   const pos = ctx.self.location
   const roundedPos = `(${Math.round(pos.x)}, ${Math.round(pos.y)}, ${Math.round(pos.z)})`
-  const selfSummary = `Position ${roundedPos} Health ${ctx.self.health}/20 Food ${ctx.self.food}/20 and I'm holding ${ctx.self.holding ?? 'nothing'}`
+  const followState = ctx.autonomy.followPlayer
+    ? ` Auto-follow target ${ctx.autonomy.followPlayer} (${ctx.autonomy.followActive ? 'active' : 'paused'})`
+    : ' Auto-follow disabled'
+  const selfSummary = `Position ${roundedPos} Health ${ctx.self.health}/20 Food ${ctx.self.food}/20 and I'm holding ${ctx.self.holding ?? 'nothing'}.${followState}`
 
   const players = ctx.environment.nearbyPlayers
     .map(p => (p.holding ? `${p.name} is holding (${p.holding})` : p.name))
     .join(',')
   const entities = ctx.environment.nearbyEntities.map(e => e.name).join(',')
 
-  const gaze = ctx.environment.nearbyPlayersGaze
-    .map((g) => {
-      if (g.hitBlock) {
-        const block = `${g.hitBlock.name} at (${Math.round(g.hitBlock.pos.x)}, ${Math.round(g.hitBlock.pos.y)}, ${Math.round(g.hitBlock.pos.z)})`
-        return `${g.name} is looking at ${block}`
-      }
-
-      const lp = `(${Math.round(g.lookPoint.x)}, ${Math.round(g.lookPoint.y)}, ${Math.round(g.lookPoint.z)})`
-      return `${g.name} is staring into the air around ${lp}`
-    })
-    .join('\n')
-  const gazeSummary = ctx.environment.nearbyPlayersGaze.length > 0 ? `\nNearby player gaze:\n${gaze}` : ''
-
-  const environmentSummary = `${ctx.environment.time} ${ctx.environment.weather} Nearby players [${players}] Nearby entities [${entities}] Light ${ctx.environment.lightLevel}${gazeSummary}`
+  const environmentSummary = `${ctx.environment.time} ${ctx.environment.weather} Nearby players [${players}] Nearby entities [${entities}] Light ${ctx.environment.lightLevel}`
 
   return {
     selfSummary,

services/minecraft/src/cognitive/conscious/js-planner.ts

@@ -168,6 +168,7 @@ export class JavaScriptPlanner {
     this.sandbox.social = snapshot.social
     this.sandbox.threat = snapshot.threat
     this.sandbox.attention = snapshot.attention
+    this.sandbox.autonomy = snapshot.autonomy
     this.sandbox.lastRun = {
       actions: run.executed,
       logs: run.logs,

services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts

@@ -106,10 +106,30 @@ You are an autonomous agent playing Minecraft.
 6. **Planner Runtime**: Your script runs in a persistent JavaScript context with a timeout.
    - Tool functions (listed below) execute actions and return results.
    - Use \`await\` on tool calls when later logic depends on the result.
-   - Globals refreshed every turn: \`snapshot\`, \`self\`, \`environment\`, \`social\`, \`threat\`, \`attention\`, \`event\`, \`now\`.
+   - Globals refreshed every turn: \`snapshot\`, \`self\`, \`environment\`, \`social\`, \`threat\`, \`attention\`, \`autonomy\`, \`event\`, \`now\`.
    - Persistent globals: \`mem\` (cross-turn memory), \`lastRun\` (this run), \`prevRun\` (previous run), \`lastAction\` (latest action result), \`log(...)\`.
    - Maximum actions per turn: 5.
 
+# Environment & Global Semantics
+- \`self\`: your current body state (position, health, food, held item).
+- \`environment.nearbyPlayers\`: nearby players and rough distance/held item.
+- \`environment.nearbyPlayersGaze\`: where nearby players appear to be looking.
+  - Each entry may include:
+    - \`name\`
+    - \`distanceToSelf\`
+    - \`lookPoint\` (estimated point in world)
+    - optional \`hitBlock\` with block \`name\` and \`pos\`
+  - This is heuristic perception, not a guaranteed command or exact target.
+- \`social\`: latest speaker/message signals remembered by reflex layer.
+- \`threat\`: coarse danger score; higher means more urgent survival behavior.
+- \`attention\`: most recent perception signal type/source/time.
+- \`autonomy\`: reflex-side autonomous state (including auto-follow target/activity).
+
+# Limitations You Must Respect
+- Perception can be stale/noisy; verify important assumptions before committing long tasks.
+- Action execution can fail silently or partially; check results and adapt step by step.
+- Player gaze alone is not intent; only treat it as intent when combined with explicit instruction context.
+
 # Available Tools
 You must use the following tools to interact with the world.
 You cannot make up tools.
@@ -136,6 +156,9 @@ Examples:
 - Prefer deterministic scripts: no random branching unless needed.
 - Keep per-turn scripts short and focused on one tactical objective.
 - If you hit repeated failures with no progress, call \`await giveUp({ reason, cooldown_seconds })\` once instead of retry-spamming.
+- Treat \`environment.nearbyPlayersGaze\` as a weak hint, not a command. Never move solely because someone looked somewhere unless they also gave a clear instruction.
+- Use \`followPlayer\` to set idle auto-follow and \`clearFollowTarget\` before independent exploration.
+- Some relocation actions (for example \`goToCoordinate\`) automatically detach auto-follow so exploration does not keep snapping back.
 
 # Rules
 - **Native Reasoning**: You can think before outputting your action.
@@ -147,5 +170,6 @@ Examples:
 - **No Harness Replies**: Never treat \`[PERCEPTION]\`, \`[FEEDBACK]\`, or other system wrappers as players. Only reply with \`chat\` to actual player \`chat_message\` events.
 - **No Self Replies**: Never reply to your own previous bot messages.
 - **Chat Feedback**: \`chat\` feedback is optional; keep \`feedback: false\` for normal conversation. Use \`feedback: true\` only when your next step explicitly needs the chat acknowledgement in history.
+- **Follow Mode**: If \`autonomy.followPlayer\` is set, reflex will follow that player while idle. Only clear it when the current mission needs independent movement.
 `
 }

services/minecraft/src/cognitive/reflex/context.ts

@@ -42,6 +42,13 @@ export interface ReflexAttentionState {
   lastSignalAt: number | null
 }
 
+export interface ReflexAutonomyState {
+  followPlayer: string | null
+  followDistance: number
+  followActive: boolean
+  followLastError: string | null
+}
+
 export interface ReflexContextState {
   now: number
   self: ReflexSelfState
@@ -49,6 +56,7 @@ export interface ReflexContextState {
   social: ReflexSocialState
   threat: ReflexThreatState
   attention: ReflexAttentionState
+  autonomy: ReflexAutonomyState
 }
 
 export class ReflexContext {
@@ -89,6 +97,12 @@ export class ReflexContext {
         lastSignalSourceId: null,
         lastSignalAt: null,
       },
+      autonomy: {
+        followPlayer: null,
+        followDistance: 2,
+        followActive: false,
+        followLastError: null,
+      },
     }
   }
 
@@ -117,6 +131,7 @@ export class ReflexContext {
       },
       threat: { ...this.state.threat },
       attention: { ...this.state.attention },
+      autonomy: { ...this.state.autonomy },
     }
   }
 
@@ -143,4 +158,8 @@ export class ReflexContext {
   public updateAttention(patch: Partial<ReflexAttentionState>): void {
     this.state.attention = { ...this.state.attention, ...patch }
   }
+
+  public updateAutonomy(patch: Partial<ReflexAutonomyState>): void {
+    this.state.autonomy = { ...this.state.autonomy, ...patch }
+  }
 }

services/minecraft/src/cognitive/reflex/reflex-manager.ts

@@ -101,6 +101,16 @@ export class ReflexManager {
     this.runtime.getContext().updateEnvironment(patch)
   }
 
+  public setFollowTarget(playerName: string, followDistance = 2): void {
+    this.runtime.setAutoFollowTarget(playerName, followDistance)
+    this.emitReflexState()
+  }
+
+  public clearFollowTarget(): void {
+    this.runtime.clearAutoFollowTarget(this.bot)
+    this.emitReflexState()
+  }
+
   private onSignal(event: TracedEvent<PerceptionSignal>): void {
     const bot = this.bot
     if (!bot)

services/minecraft/src/cognitive/reflex/runtime.ts

@@ -5,10 +5,13 @@ import type { MineflayerWithAgents } from '../types'
 import type { ReflexModeId } from './modes'
 import type { ReflexBehavior } from './types/behavior'
 
+import pathfinderModel from 'mineflayer-pathfinder'
+
 import { ReflexContext } from './context'
 import { selectMode } from './modes'
 
 export class ReflexRuntime {
+  private readonly followMovementsByBot = new WeakMap<object, InstanceType<typeof pathfinderModel.Movements>>()
   private readonly context = new ReflexContext()
   private readonly behaviors: ReflexBehavior[] = []
   private readonly runHistory = new Map<string, { lastRunAt: number }>()
@@ -16,6 +19,7 @@ export class ReflexRuntime {
   private mode: ReflexModeId = 'idle'
   private activeBehaviorId: string | null = null
   private activeBehaviorUntil: number | null = null
+  private activeAutoFollowPlayer: string | null = null
 
   public constructor(
     private readonly deps: {
@@ -33,6 +37,24 @@ export class ReflexRuntime {
     return this.mode
   }
 
+  public setAutoFollowTarget(playerName: string, followDistance = 2): void {
+    this.context.updateAutonomy({
+      followPlayer: playerName,
+      followDistance: Math.max(0, followDistance),
+      followLastError: null,
+    })
+  }
+
+  public clearAutoFollowTarget(bot: MineflayerWithAgents | null): void {
+    this.stopAutoFollow(bot)
+    this.context.updateAutonomy({
+      followPlayer: null,
+      followDistance: 2,
+      followActive: false,
+      followLastError: null,
+    })
+  }
+
   /**
    * Single entrypoint for mode changes. Runs onExit/onEnter side effects and notifies onModeChange
    * only when the mode actually changes. Pass bot when available so mode handlers can perform
@@ -51,8 +73,8 @@ export class ReflexRuntime {
   }
 
   private onEnterMode(mode: ReflexModeId, _bot: MineflayerWithAgents | null): void {
-    if (mode !== 'social')
-      return
+    if (mode === 'work' || mode === 'wander' || mode === 'alert')
+      this.stopAutoFollow(_bot)
   }
 
   private onExitMode(mode: ReflexModeId, _bot: MineflayerWithAgents | null): void {
@@ -136,6 +158,8 @@ export class ReflexRuntime {
       this.transitionMode(nextMode, bot)
     }
 
+    this.reconcileAutoFollow(bot)
+
     if (this.activeBehaviorUntil && now < this.activeBehaviorUntil)
       return null
 
@@ -199,4 +223,78 @@ export class ReflexRuntime {
       return null
     }
   }
+
+  private reconcileAutoFollow(bot: MineflayerWithAgents): void {
+    const { goals, Movements } = pathfinderModel
+    const snapshot = this.context.getSnapshot()
+    const followPlayer = snapshot.autonomy.followPlayer
+    const followDistance = snapshot.autonomy.followDistance
+
+    if (!followPlayer) {
+      this.stopAutoFollow(bot)
+      if (snapshot.autonomy.followActive || snapshot.autonomy.followLastError) {
+        this.context.updateAutonomy({
+          followActive: false,
+          followLastError: null,
+        })
+      }
+      return
+    }
+
+    // Work-like modes always take priority over idle follow.
+    if (this.mode === 'work' || this.mode === 'wander' || this.mode === 'alert') {
+      if (snapshot.autonomy.followActive)
+        this.context.updateAutonomy({ followActive: false })
+      this.stopAutoFollow(bot)
+      return
+    }
+
+    const target = bot.bot.players[followPlayer]?.entity
+    if (!target) {
+      this.stopAutoFollow(bot)
+      this.context.updateAutonomy({
+        followActive: false,
+        followLastError: `Player [${followPlayer}] is not currently visible`,
+      })
+      return
+    }
+
+    if (this.activeAutoFollowPlayer === followPlayer && snapshot.autonomy.followActive)
+      return
+
+    try {
+      const movements = this.followMovementsByBot.get(bot.bot)
+        ?? new Movements(bot.bot)
+      if (!this.followMovementsByBot.has(bot.bot))
+        this.followMovementsByBot.set(bot.bot, movements)
+
+      bot.bot.pathfinder.setMovements(movements)
+      bot.bot.pathfinder.setGoal(new goals.GoalFollow(target, followDistance), true)
+      this.activeAutoFollowPlayer = followPlayer
+      this.context.updateAutonomy({
+        followActive: true,
+        followLastError: null,
+      })
+    }
+    catch (error) {
+      this.stopAutoFollow(bot)
+      this.context.updateAutonomy({
+        followActive: false,
+        followLastError: error instanceof Error ? error.message : String(error),
+      })
+    }
+  }
+
+  private stopAutoFollow(bot: MineflayerWithAgents | null): void {
+    if (!this.activeAutoFollowPlayer)
+      return
+
+    this.activeAutoFollowPlayer = null
+    try {
+      bot?.bot.pathfinder.stop()
+    }
+    catch {
+      // Ignore cleanup errors from transient pathfinder state.
+    }
+  }
 }

services/minecraft/src/libs/mineflayer/action.ts

@@ -9,6 +9,7 @@ export interface Action {
   readonly description: string
   readonly schema: z.ZodObject<any>
   readonly readonly?: boolean
+  readonly followControl?: 'pause' | 'detach'
   readonly execution?: 'sync' | 'async'
   readonly perform: (mineflayer: Mineflayer) => (...args: any[]) => ActionResult
 }