From e203ecea969eec1f2df92c9f24b8c463662b0a8a Mon Sep 17 00:00:00 2001
From: Neko Ayaka <neko@ayaka.moe>
Date: Thu, 16 Jan 2025 16:27:42 +0800
Subject: [PATCH] feat: integrated as Airi client

---
 services/minecraft/src/main.ts                |  5 +-
 .../minecraft/src/mineflayer/llm-agent.ts     | 47 ++++++++++++++++++-
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/services/minecraft/src/main.ts b/services/minecraft/src/main.ts
index 3e28c5369..138d1b162 100644
--- a/services/minecraft/src/main.ts
+++ b/services/minecraft/src/main.ts
@@ -1,6 +1,7 @@
 import process, { exit } from 'node:process'
 
 import { useLogg } from '@guiiai/logg'
+import { Client } from '@proj-airi/server-sdk'
 import MineflayerArmorManager from 'mineflayer-armor-manager'
 import { loader as MineflayerAutoEat } from 'mineflayer-auto-eat'
 import { plugin as MineflayerCollectBlock } from 'mineflayer-collectblock'
@@ -33,9 +34,11 @@ async function main() {
     ],
   })
 
+  const airiClient = new Client({ name: 'minecraft-bot', url: 'ws://localhost:6121/ws' })
+
   // Dynamically load LLMAgent after bot is initialized
   const agent = await initAgent(bot)
-  await bot.loadPlugin(LLMAgent({ agent }))
+  await bot.loadPlugin(LLMAgent({ agent, airiClient }))
 
   process.on('SIGINT', () => {
     bot.stop()
diff --git a/services/minecraft/src/mineflayer/llm-agent.ts b/services/minecraft/src/mineflayer/llm-agent.ts
index 18aab2c4a..696ef642b 100644
--- a/services/minecraft/src/mineflayer/llm-agent.ts
+++ b/services/minecraft/src/mineflayer/llm-agent.ts
@@ -1,13 +1,14 @@
+import type { Client } from '@proj-airi/server-sdk'
 import type { Neuri } from 'neuri'
-import type { MineflayerPlugin } from '../libs/mineflayer/plugin'
 
+import type { MineflayerPlugin } from '../libs/mineflayer/plugin'
 import { useLogg } from '@guiiai/logg'
 import { assistant, system, user } from 'neuri/openai'
 import { toRetriable } from 'src/utils/reliability'
 import { formBotChat } from '../libs/mineflayer/message'
 import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent'
 
-export function LLMAgent(options: { agent: Neuri }): MineflayerPlugin {
+export function LLMAgent(options: { agent: Neuri, airiClient: Client }): MineflayerPlugin {
   return {
     async created(bot) {
       const agent = options.agent
@@ -59,6 +60,48 @@ export function LLMAgent(options: { agent: Neuri }): MineflayerPlugin {
         }
       })
 
+      options.airiClient.onEvent('input:text:voice', async (event) => {
+        logger.withFields({ user: event.data.discord?.guildMember, message: event.data.transcription }).log('Chat message received')
+
+        const statusPrompt = await genStatusPrompt(bot)
+        bot.memory.chatHistory.push(system(statusPrompt))
+        bot.memory.chatHistory.push(user(`${'NekoMeowww'}: ${event.data.transcription}`))
+
+        // logger.withFields({ chatHistory: bot.memory.chatHistory }).log('chatHistory')
+        logger.withFields({ statusPrompt }).log('statusPrompt')
+
+        const content = await agent.handleStateless([...bot.memory.chatHistory], async (c) => {
+          logger.log('thinking...')
+
+          const handleCompletion = async (c: any): Promise<string> => {
+            const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) || { error: { message: 'Unknown error' } }
+            if (!completion || 'error' in completion) {
+              logger.withFields(c).error('Completion')
+              throw new Error(completion?.error?.message ?? 'Unknown error')
+            }
+
+            const content = await completion?.firstContent()
+            logger.withFields({ usage: completion.usage, content }).log('output')
+            bot.memory.chatHistory.push(assistant(content))
+
+            return content
+          }
+
+          const retirableHandler = toRetriable<any, string>(
+            3, // retryLimit
+            1000, // delayInterval in ms
+            handleCompletion,
+          )
+
+          return await retirableHandler(c)
+        })
+
+        if (content) {
+          logger.withFields({ content }).log('responded')
+          bot.bot.chat(content)
+        }
+      })
+
       bot.bot.on('chat', onChat)
     },
   }