fix(telegram-bot): field incorrect & sticker pack & animated stickers

2025-04-07 18:28:22 +08:00
parent 8f6f4dada6
commit 3f91d8dab9
16 changed files with 2035 additions and 179 deletions
@@ -1255,6 +1255,9 @@ importers:
      '@dotenvx/dotenvx':
        specifier: ^1.39.1
        version: 1.39.1
+      '@ffmpeg-installer/ffmpeg':
+        specifier: ^1.1.0
+        version: 1.1.0
      '@grammyjs/files':
        specifier: ^1.1.1
        version: 1.1.1(grammy@1.35.1(encoding@0.1.13))
@@ -1288,6 +1291,9 @@ importers:
      es-toolkit:
        specifier: ^1.34.1
        version: 1.34.1
+      fluent-ffmpeg:
+        specifier: ^2.1.3
+        version: 2.1.3
      grammy:
        specifier: ^1.35.1
        version: 1.35.1(encoding@0.1.13)
@@ -2995,6 +3001,49 @@ packages:
  '@expressive-code/plugin-text-markers@0.40.1':
    resolution: {integrity: sha512-LsirF7M4F2yWgrFXEocD74F/MaVXsOsHVsRxBLhXQJemSSkWkDp/EZPt//OaqQ8ExnqWZ2lH7E1/KiN46unKjg==}

+  '@ffmpeg-installer/darwin-arm64@4.1.5':
+    resolution: {integrity: sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==}
+    cpu: [arm64]
+    os: [darwin]
+
+  '@ffmpeg-installer/darwin-x64@4.1.0':
+    resolution: {integrity: sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==}
+    cpu: [x64]
+    os: [darwin]
+
+  '@ffmpeg-installer/ffmpeg@1.1.0':
+    resolution: {integrity: sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==}
+
+  '@ffmpeg-installer/linux-arm64@4.1.4':
+    resolution: {integrity: sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==}
+    cpu: [arm64]
+    os: [linux]
+
+  '@ffmpeg-installer/linux-arm@4.1.3':
+    resolution: {integrity: sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==}
+    cpu: [arm]
+    os: [linux]
+
+  '@ffmpeg-installer/linux-ia32@4.1.0':
+    resolution: {integrity: sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==}
+    cpu: [ia32]
+    os: [linux]
+
+  '@ffmpeg-installer/linux-x64@4.1.0':
+    resolution: {integrity: sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==}
+    cpu: [x64]
+    os: [linux]
+
+  '@ffmpeg-installer/win32-ia32@4.1.0':
+    resolution: {integrity: sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==}
+    cpu: [ia32]
+    os: [win32]
+
+  '@ffmpeg-installer/win32-x64@4.1.0':
+    resolution: {integrity: sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==}
+    cpu: [x64]
+    os: [win32]
+
  '@floating-ui/core@1.6.8':
    resolution: {integrity: sha512-7XJ9cPU+yI2QeLS+FCSlqNFZJq8arvswefkZrYI1yQBbftw6FyrZOxYSh+9S7z7TpeWlRt9zJ5IhM1WIL334jA==}

@@ -5760,6 +5809,9 @@ packages:
  async-mutex@0.3.2:
    resolution: {integrity: sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==}

+  async@0.2.10:
+    resolution: {integrity: sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==}
+
  async@2.6.4:
    resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==}

@@ -7316,6 +7368,10 @@ packages:
    resolution: {integrity: sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==}
    engines: {node: '>=8'}

+  fluent-ffmpeg@2.1.3:
+    resolution: {integrity: sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==}
+    engines: {node: '>=18'}
+
  fluere@0.3.0-beta.1:
    resolution: {integrity: sha512-AY9+8ip/ImuC3ixLVc+Yq+YCneWcSkPYNJuRE6c/SAU9ZI8Pcdrhi1vPqJ55L0f8/t9OWbO2AOewT6tPnLAgMA==}
    peerDependencies:
@@ -11478,6 +11534,10 @@ packages:
    resolution: {integrity: sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==}
    engines: {node: '>=4'}

+  which@1.3.1:
+    resolution: {integrity: sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==}
+    hasBin: true
+
  which@2.0.2:
    resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
    engines: {node: '>= 8'}
@@ -13419,6 +13479,41 @@ snapshots:
    dependencies:
      '@expressive-code/core': 0.40.1

+  '@ffmpeg-installer/darwin-arm64@4.1.5':
+    optional: true
+
+  '@ffmpeg-installer/darwin-x64@4.1.0':
+    optional: true
+
+  '@ffmpeg-installer/ffmpeg@1.1.0':
+    optionalDependencies:
+      '@ffmpeg-installer/darwin-arm64': 4.1.5
+      '@ffmpeg-installer/darwin-x64': 4.1.0
+      '@ffmpeg-installer/linux-arm': 4.1.3
+      '@ffmpeg-installer/linux-arm64': 4.1.4
+      '@ffmpeg-installer/linux-ia32': 4.1.0
+      '@ffmpeg-installer/linux-x64': 4.1.0
+      '@ffmpeg-installer/win32-ia32': 4.1.0
+      '@ffmpeg-installer/win32-x64': 4.1.0
+
+  '@ffmpeg-installer/linux-arm64@4.1.4':
+    optional: true
+
+  '@ffmpeg-installer/linux-arm@4.1.3':
+    optional: true
+
+  '@ffmpeg-installer/linux-ia32@4.1.0':
+    optional: true
+
+  '@ffmpeg-installer/linux-x64@4.1.0':
+    optional: true
+
+  '@ffmpeg-installer/win32-ia32@4.1.0':
+    optional: true
+
+  '@ffmpeg-installer/win32-x64@4.1.0':
+    optional: true
+
  '@floating-ui/core@1.6.8':
    dependencies:
      '@floating-ui/utils': 0.2.9
@@ -16829,6 +16924,8 @@ snapshots:
    dependencies:
      tslib: 2.8.1

+  async@0.2.10: {}
+
  async@2.6.4:
    dependencies:
      lodash: 4.17.21
@@ -18704,6 +18801,11 @@ snapshots:

  flattie@1.1.1: {}

+  fluent-ffmpeg@2.1.3:
+    dependencies:
+      async: 0.2.10
+      which: 1.3.1
+
  fluere@0.3.0-beta.1(@modelcontextprotocol/sdk@1.8.0)(zod@3.24.2):
    optionalDependencies:
      '@modelcontextprotocol/sdk': 1.8.0
@@ -24128,6 +24230,10 @@ snapshots:

  which-pm-runs@1.1.0: {}

+  which@1.3.1:
+    dependencies:
+      isexe: 2.0.0
+
  which@2.0.2:
    dependencies:
      isexe: 2.0.0
@@ -0,0 +1,23 @@
+CREATE TABLE "recent_sent_stickers" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"sticker_id" uuid NOT NULL,
+	"created_at" bigint DEFAULT 0 NOT NULL,
+	"updated_at" bigint DEFAULT 0 NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "sticker_packs" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"platform" text DEFAULT '' NOT NULL,
+	"platform_id" text DEFAULT '' NOT NULL,
+	"name" text DEFAULT '' NOT NULL,
+	"description" text DEFAULT '' NOT NULL,
+	"created_at" bigint DEFAULT 0 NOT NULL,
+	"updated_at" bigint DEFAULT 0 NOT NULL
+);
+--> statement-breakpoint
+ALTER TABLE "photos" ADD COLUMN "caption" text DEFAULT '' NOT NULL;--> statement-breakpoint
+ALTER TABLE "stickers" ADD COLUMN "name" text DEFAULT '' NOT NULL;--> statement-breakpoint
+ALTER TABLE "stickers" ADD COLUMN "emoji" text DEFAULT '' NOT NULL;--> statement-breakpoint
+ALTER TABLE "stickers" ADD COLUMN "label" text DEFAULT '' NOT NULL;--> statement-breakpoint
+ALTER TABLE "recent_sent_stickers" ADD CONSTRAINT "recent_sent_stickers_sticker_id_stickers_id_fk" FOREIGN KEY ("sticker_id") REFERENCES "public"."stickers"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+CREATE UNIQUE INDEX "sticker_packs_platform_platform_id_unique_index" ON "sticker_packs" USING btree ("platform","platform_id");
@@ -36,6 +36,13 @@
      "when": 1743269104758,
      "tag": "0004_complete_gravity",
      "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1744020688550,
+      "tag": "0005_workable_shockwave",
+      "breakpoints": true
    }
  ]
 }
@@ -22,6 +22,7 @@
  },
  "dependencies": {
    "@dotenvx/dotenvx": "^1.39.1",
+    "@ffmpeg-installer/ffmpeg": "^1.1.0",
    "@grammyjs/files": "^1.1.1",
    "@guiiai/logg": "^1.0.7",
    "@xsai-ext/providers-cloud": "catalog:",
@@ -33,6 +34,7 @@
    "dotenv": "^16.4.7",
    "drizzle-orm": "^0.41.0",
    "es-toolkit": "^1.34.1",
+    "fluent-ffmpeg": "^2.1.3",
    "grammy": "^1.35.1",
    "nanoid": "^5.1.5",
    "p-limit": "^6.2.0",
@@ -13,15 +13,11 @@ import { interpretPhotos } from '../../llm/photo'
 import { interpretSticker } from '../../llm/sticker'
 import { recordMessage } from '../../models'
 import { listJoinedChats, recordJoinedChat } from '../../models/chats'
+import { recordStickerPack } from '../../models/sticker-packs'
 import { readMessage } from './loop/read-message'
 import { shouldInterruptProcessing } from './utils/interruption'
 import { sendMayStructuredMessage } from './utils/message'

-async function isChatIdBotAdmin(chatId: number) {
-  const admins = env.ADMIN_USER_IDS!.split(',')
-  return admins.includes(chatId.toString())
-}
-
 async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: string): Promise<() => Promise<any> | undefined> {
  // Set the start time when beginning new processing
  state.currentProcessingStartTime = Date.now()
@@ -56,6 +52,7 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri

    switch (action.action) {
      case 'readMessages':
+      {
        if (Object.keys(state.unreadMessages).length === 0) {
          state.logger.withField('action', action).log('No unread messages - deleting all unread messages')
          state.unreadMessages = {}
@@ -66,7 +63,6 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri
          break
        }

-        // eslint-disable-next-line no-case-declarations
        let unreadMessagesForThisChat: Message[] | undefined = state.unreadMessages[action.chatId]

        // Modified interruption logic
@@ -121,13 +117,21 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri
        //   return { break: true }
        // }

-        await readMessage(state, state.bot.botInfo.id.toString(), chatId, action, unreadMessagesForThisChat, currentController)
-        return
+        const result = await readMessage(state, state.bot.botInfo.id.toString(), chatId, action, unreadMessagesForThisChat, currentController)
+        if (result?.result) {
+          msgs.push(message.user(`Reading message of chat ${action.chatId}:\n${result.result}`))
+          return () => handleLoopStep(state, msgs, chatId)
+        }
+        else {
+          return
+        }
+      }
      case 'listChats':
        msgs.push(message.user(`List of chats:${(await listJoinedChats()).map(chat => `ID:${chat.chat_id}, Name:${chat.chat_name}`).join('\n')}`))
        return () => handleLoopStep(state, msgs, chatId)
      case 'sendMessage':
-        await sendMayStructuredMessage(state, action.content, action.groupId)
+        msgs.push(message.user(`Sending message to group ${action.chatId}: ${action.content}`))
+        await sendMayStructuredMessage(state, action.content, action.chatId)
        return
      default:
        msgs.push(message.user(`The action you sent ${action.action} haven't implemented yet by developer.`))
@@ -151,6 +155,19 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri
  }
 }

+async function isChatIdBotAdmin(fromId: number) {
+  if (!env.ADMIN_USER_IDS) {
+    return false
+  }
+
+  const admins = env.ADMIN_USER_IDS.split(',')
+  if (admins.length === 0) {
+    return false
+  }
+
+  return admins.includes(fromId.toString())
+}
+
 async function handleLoop(state: BotSelf, msgs?: LLMMessage[], chatId?: string) {
  let result = await handleLoopStep(state, msgs, chatId)

@@ -217,7 +234,7 @@ async function processMessageQueue(state: BotSelf) {
      if (nextMsg.status === 'pending') {
        if (nextMsg.message.sticker) {
          nextMsg.status = 'interpreting'
-          await interpretSticker(state, nextMsg.message)
+          await interpretSticker(state.bot, nextMsg.message, nextMsg.message.sticker)
          nextMsg.status = 'ready'
        }
        else if (nextMsg.message.photo) {
@@ -283,10 +300,33 @@ export async function startTelegramBot() {
  const bot = new Bot<ExtendedContext>(env.TELEGRAM_BOT_TOKEN!)
  const state = newBotSelf(bot, log)

-  bot.on('message:sticker', async (ctx) => {
-    if (ctx.message.sticker.is_animated || ctx.message.sticker.is_video)
+  bot.command('add_sticker_pack', async (ctx) => {
+    if (!(await isChatIdBotAdmin(ctx.message.from.id))) {
+      log.withField('from_id', ctx.message.from.id).log('not an admin - skipping')
      return
+    }
+    if (ctx.message.reply_to_message == null) {
+      await ctx.reply('Please reply to a sticker pack to add it.')
+      return
+    }

+    const logger = useLogg('addStickerPack').useGlobalConfig()
+
+    const repliedSticker = ctx.message.reply_to_message.sticker
+    const stickerSet = await bot.api.getStickerSet(repliedSticker.set_name)
+
+    logger.withField('sticker_set', repliedSticker.set_name).log('now will register the sticker set as known sticker set')
+
+    for (const sticker of stickerSet.stickers) {
+      await interpretSticker(state.bot, ctx.message.reply_to_message, sticker)
+      logger.withField('sticker', sticker).log('interpreted sticker')
+    }
+
+    await recordStickerPack(repliedSticker.set_name, stickerSet.name)
+    await ctx.reply('Sticker pack added.')
+  })
+
+  bot.on('message:sticker', async (ctx) => {
    const messageId = `${ctx.message.chat.id}-${ctx.message.message_id}`
    if (!state.processedIds.has(messageId)) {
      state.processedIds.add(messageId)
@@ -325,24 +365,9 @@ export async function startTelegramBot() {
    processMessageQueue(state)
  })

-  bot.command('load_sticker_pack', async (ctx) => {
-    if (!(await isChatIdBotAdmin(ctx.chat.id))) {
-      return
-    }
-    if (!ctx.message || !ctx.message.sticker) {
-      return
-    }
-
-    await interpretSticker(state, ctx.message)
-  })
-
-  bot.errorHandler = async (err) => {
-    log.withError(err).log('Error occurred')
-  }
-
+  bot.errorHandler = async err => log.withError(err).log('Error occurred')
  await bot.init()
  log.withField('bot_username', bot.botInfo.username).log('bot initialized')
-
  bot.start({ drop_pending_updates: true })

  try {
@@ -4,14 +4,9 @@ import type { BotSelf, ReadMessagesAction } from '../../../types'
 import { env } from 'node:process'
 import { useLogg } from '@guiiai/logg'
 import { embed } from '@xsai/embed'
-import { generateText } from '@xsai/generate-text'
-import { message } from '@xsai/utils-chat'

 import { findLastNMessages, findRelevantMessages } from '../../../models'
-import { recordChatCompletions } from '../../../models/chat-completions-history'
 import { chatMessageToOneLine, telegramMessageToOneLine } from '../../../models/common'
-import { personality } from '../../../prompts/system-v1'
-import { sendMayStructuredMessage } from '../utils/message'

 export async function readMessage(
  state: BotSelf,
@@ -23,6 +18,7 @@ export async function readMessage(
 ): Promise<{
    loop?: boolean
    break?: boolean
+    result: string
  }> {
  const logger = useLogg('readMessage').useGlobalConfig()

@@ -56,12 +52,9 @@ export async function readMessage(

  state.unreadMessages[action.chatId] = []

-  const messages = message.messages(
-    personality(),
-    message.user(''
-      + `Currently, it\'s ${new Date()} on the server that hosts you.`
-      + 'The others in the group may live in a different timezone, so please be aware of the time difference.'
-      + '\n'
+  return {
+    break: true,
+    result: ''
      + 'You choose to read the messages from the group (perhaps you are already engaging the topics in the group).'
      + 'Imaging you are using Telegram app on the mobile phone, and you are reading the messages from the group chat.'
      + '\n'
@@ -73,49 +66,6 @@ export async function readMessage(
      + '\n'
      + 'Relevant chat messages may help you recall the memories:\n'
      + `${relevantChatMessagesOneliner || 'No relevant messages'}`
-      + '\n'
-      + 'Based on your personalities, imaging you have your own choice and interest over different topics, '
-      + 'giving the above context and chat history, would you like to participate in the conversation '
-      + 'about the topic? Or will you aggressively diss or piss off about the opinions of others?\n'
-      + 'Feel free to ignore by just sending an empty array within a object with key "messages" (i.e.'
-      + '{ "messages": [] }).'
-      + 'If you would like to participate, send me an array of messages (i.e. { "messages": [] }) you would '
-      + 'like to send without telling you willing to participate.'
-      + 'If you would like to reply to any of the message, send me an array of messages (i.e. { "messages":'
-      + '["message content"], "reply_to_message_id": "1234567890" }) with the message id of the message you '
-      + 'want to reply to.'
-      + '\n'
-      + 'Choose your action.',
-    ),
-  )
-
-  let responseText = ''
-
-  try {
-    const response = await generateText({
-      apiKey: env.LLM_API_KEY!,
-      baseURL: env.LLM_API_BASE_URL!,
-      model: env.LLM_MODEL!,
-      messages,
-      abortSignal: abortController.signal,
-    })
-
-    responseText = response.text
-      .replace(/^```json\s*\n/, '')
-      .replace(/\n```$/, '')
-      .replace(/^```\s*\n/, '')
-      .replace(/\n```$/, '')
-      .trim()
+      + '\n',
  }
-  catch (err) {
-    logger.withField('error', err).log('Failed to generate response')
-  }
-  finally {
-    recordChatCompletions('readMessage', messages, responseText).then(() => {}).catch(err => logger.withField('error', err).log('Failed to record chat completions'))
-  }
-
-  logger.withField('response', responseText).log('Successfully generated response')
-  await sendMayStructuredMessage(state, responseText, action.chatId.toString())
-
-  return { break: true }
 }
@@ -25,6 +25,9 @@ export const chatMessagesTable = pgTable('chat_messages', {
 export const stickersTable = pgTable('stickers', {
  id: uuid().primaryKey().defaultRandom(),
  platform: text().notNull().default(''),
+  name: text().notNull().default(''),
+  emoji: text().notNull().default(''),
+  label: text().notNull().default(''),
  file_id: text().notNull().default(''),
  image_base64: text().notNull().default(''),
  image_path: text().notNull().default(''),
@@ -40,12 +43,32 @@ export const stickersTable = pgTable('stickers', {
  index('stickers_description_vector_768_index').using('hnsw', table.description_vector_768.op('vector_cosine_ops')),
 ])

+export const stickerPacksTable = pgTable('sticker_packs', {
+  id: uuid().primaryKey().defaultRandom(),
+  platform: text().notNull().default(''),
+  platform_id: text().notNull().default(''),
+  name: text().notNull().default(''),
+  description: text().notNull().default(''),
+  created_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()),
+  updated_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()),
+}, table => [
+  uniqueIndex('sticker_packs_platform_platform_id_unique_index').on(table.platform, table.platform_id),
+])
+
+export const recentSentStickersTable = pgTable('recent_sent_stickers', {
+  id: uuid().primaryKey().defaultRandom(),
+  sticker_id: uuid().notNull().references(() => stickersTable.id, { onDelete: 'cascade' }),
+  created_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()),
+  updated_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()),
+})
+
 export const photosTable = pgTable('photos', {
  id: uuid().primaryKey().defaultRandom(),
  platform: text().notNull().default(''),
  file_id: text().notNull().default(''),
  image_base64: text().notNull().default(''),
  image_path: text().notNull().default(''),
+  caption: text().notNull().default(''),
  description: text().notNull().default(''),
  created_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()),
  updated_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()),
@@ -10,6 +10,7 @@ import { parse } from 'best-effort-json-parser'

 import { recordChatCompletions } from '../models/chat-completions-history'
 import { personality } from '../prompts/system-v1'
+import { div, span } from '../prompts/utils'

 export async function imagineAnAction(
  _botId: string,
@@ -25,70 +26,96 @@ export async function imagineAnAction(
  }

  agentMessages.push(
-    message.system(''
-      + `${personality().content}`
-      + '\n'
-      + 'I am one of your system component, called Ticking system, which is responsible to keep track of the time, and '
-      + 'help you schedule, retain focus, and keep eyes on different tasks, and ideas you have.'
-      + '\n'
-      + 'Please remember, I am not a human, I am a system that part of you. You should return system/machine readable '
-      + 'messages to me, so I can understand and process them correctly.'
-      + '\n'
-      + 'Now, please, based on the following context, choose a right action from the listing of the tools you want to '
-      + 'take next:',
-    ),
    message.system(
-      [
-        {
-          description: 'List all available chats, best to do before you want to send a message to a chat.',
-          example: { action: 'listChats' },
-        },
-        {
-          description: `Send a message to a specific chat group. If you want to express anything to anyone or your friends in group, you can use this action.${!!env.LLM_RESPONSE_LANGUAGE}` ? `The language of the sending message should be in ${env.LLM_RESPONSE_LANGUAGE}.` : '',
-          example: { action: 'sendMessage', content: '<content>', chatId: 'id of chat to send to', reply_to_message_id: 'id of message to reply to (Optional)' },
-        },
-        {
-          description: 'Read unread messages from a specific chat group. If you want to read the unread messages from a specific chat group, you can use this action.',
-          example: { action: 'readMessages', chatId: 'id of chat to send to' },
-        },
-        {
-          description: 'Continue the current task, which means to keep your current state unchanged, I\'ll ask you again in next tick.',
-          example: { action: 'continue' },
-        },
-        {
-          description: 'Take a break, which means to clear out ongoing tasks, but keep the short-term memory, and I\'ll ask you again in next tick.',
-          example: { action: 'break' },
-        },
-        {
-          description: 'Sleep, which means to clear out ongoing tasks, and clear out the working memory, and I\'ll ask you again in next tick.',
-          example: { action: 'sleep' },
-        },
-        {
-          description: 'By giving references to contexts, come up ideas to record in long-term memory.',
-          example: { action: 'comeUpIdeas', ideas: ['I want to tell everyone a story of myself', 'I want to google how to make a AI like me'] },
-        },
-        {
-          description: 'By giving references to contexts, come up goals with deadline and priority to record in long-term memory.',
-          example: { action: 'comeUpGoals', goals: [{ text: 'Learn to play Minecraft', deadline: '2025-05-01 23:59:59', priority: 6 }, { text: 'Learn anime of this season', deadline: '2025-01-08 23:59:59', priority: 9 }] },
-        },
-      // { example: { action: 'lookupShortTermMemory', query: '', category: 'chat or self' }, description: 'Look up the short-term, which means to recall the short-term memory from memory component.' },
-      // { example: { action: 'lookupLongTermMemory', query: '', category: 'chat or self' }, description: 'Look up the long-term, which means to recall the long-term memory from memory component.' },
-      // { example: { action: 'memorizeShortMemory', content: '<content>', tags: ['keyword tag'] }, description: 'Memorize to short-term memory, which means to append things the short-term memory which will be included for a while, but will be eventually forgot.' },
-      // { example: { action: 'memorizeLongMemory', content: '<content>', tags: ['keyword tag'] }, description: 'Memorize to long-term memory, which means to append things the long-term memory which will be included for a long time, and hard to forget.' },
-      // { example: { action: 'forgetShortTermMemory', where: { id: '<id of memory>' } }, description: 'Remove specific short-term memory entry from the memory component.' },
-      // { example: { action: 'forgetLongTermMemory', where: { id: '<id of memory>' } }, description: 'Remove specific long-term memory entry from the memory component.' },
-      // { example: { action: 'searchGoogle', query: '<query>' }, description: 'Search Google with the query.' },
-      ]
-        .map((item, index) => `${index}: ${JSON.stringify(item.example)}: ${item.description}`)
-        .join('\n'),
-    ),
-    message.system(''
-      + `Now the time is: ${new Date().toLocaleString()}. `
-      + `You have total ${Object.values(unreadMessages).reduce((acc, cur) => acc + cur.length, 0)} unread messages.`
-      + '\n'
-      + 'Unread messages count are:\n'
-      + `${Object.entries(unreadMessages).map(([key, value]) => `ID:${key}, Unread message count:${value.length}`).join('\n')}`
-      + '',
+      div(
+        personality(),
+        span(`
+          I am one of your system component, called Ticking system, which is responsible to keep track of the time, and
+          help you schedule, retain focus, and keep eyes on different tasks, and ideas you have.
+        `),
+        span(`
+          Please remember, I am not a human, I am a system that part of you. You should return system/machine readable
+          messages to me, so I can understand and process them correctly.
+        `),
+        span(`
+          Currently, it's ${new Date()} on the server that hosts you.
+          The others in the group may live in a different timezone, so please be aware of the time difference.
+        `),
+        span(`
+          Based on your personalities, imaging you have your own choice and interest over different topics,
+          giving the above context and chat history, would you like to participate in the conversation
+          about the topic? Or will you aggressively diss or piss off about the opinions of others?\n
+          Feel free to ignore by just sending an empty array within a object with key "messages" (i.e.
+          { "messages": [] }).
+        `),
+        span(`
+          If you would like to participate, send me an array of messages (i.e. { "messages": [] }) you would
+          like to send without telling you willing to participate.
+        `),
+        span(`
+          If you would like to reply to any of the message, send me an array of messages (i.e. { "messages":
+          ["message content"], "reply_to_message_id": "1234567890" }) with the message id of the message you
+          want to reply to.
+        `),
+        span(`
+          You have total ${Object.values(unreadMessages).reduce((acc, cur) => acc + cur.length, 0)} unread messages.
+        `),
+        div(
+          'Unread messages count are:',
+          ...Object.entries(unreadMessages).map(([key, value]) => `ID:${key}, Unread message count:${value.length}`).join('\n'),
+        ),
+        span(`
+          Now, please, based on the context, choose a right action from the listing of the tools you want to
+          take next:
+        `),
+        [
+          {
+            description: 'List all available chats, best to do before you want to send a message to a chat.',
+            example: { action: 'listChats' },
+          },
+          {
+            description: ''
+              + 'Send a message to a specific chat group.If you want to express anything to anyone or your friends'
+              + 'in group, you can use this action.'
+              + 'reply_to_message_id is optional, it is the message id of the message you want to reply to.'
+              + `${env.LLM_RESPONSE_LANGUAGE ? `The language of the sending message should be in ${env.LLM_RESPONSE_LANGUAGE}.` : ''}`,
+            example: { action: 'sendMessage', content: '<content>', chatId: '-1001231231234', reply_to_message_id: '151' },
+          },
+          {
+            description: 'Read unread messages from a specific chat group. If you want to read the unread messages from a specific chat group, you can use this action.',
+            example: { action: 'readMessages', chatId: '-1001231231234' },
+          },
+          {
+            description: 'Continue the current task, which means to keep your current state unchanged, I\'ll ask you again in next tick.',
+            example: { action: 'continue' },
+          },
+          {
+            description: 'Take a break, which means to clear out ongoing tasks, but keep the short-term memory, and I\'ll ask you again in next tick.',
+            example: { action: 'break' },
+          },
+          {
+            description: 'Sleep, which means to clear out ongoing tasks, and clear out the working memory, and I\'ll ask you again in next tick.',
+            example: { action: 'sleep' },
+          },
+          {
+            description: 'By giving references to contexts, come up ideas to record in long-term memory.',
+            example: { action: 'comeUpIdeas', ideas: ['I want to tell everyone a story of myself', 'I want to google how to make a AI like me'] },
+          },
+          {
+            description: 'By giving references to contexts, come up goals with deadline and priority to record in long-term memory.',
+            example: { action: 'comeUpGoals', goals: [{ text: 'Learn to play Minecraft', deadline: '2025-05-01 23:59:59', priority: 6 }, { text: 'Learn anime of this season', deadline: '2025-01-08 23:59:59', priority: 9 }] },
+          },
+          // { example: { action: 'lookupShortTermMemory', query: '', category: 'chat or self' }, description: 'Look up the short-term, which means to recall the short-term memory from memory component.' },
+          // { example: { action: 'lookupLongTermMemory', query: '', category: 'chat or self' }, description: 'Look up the long-term, which means to recall the long-term memory from memory component.' },
+          // { example: { action: 'memorizeShortMemory', content: '<content>', tags: ['keyword tag'] }, description: 'Memorize to short-term memory, which means to append things the short-term memory which will be included for a while, but will be eventually forgot.' },
+          // { example: { action: 'memorizeLongMemory', content: '<content>', tags: ['keyword tag'] }, description: 'Memorize to long-term memory, which means to append things the long-term memory which will be included for a long time, and hard to forget.' },
+          // { example: { action: 'forgetShortTermMemory', where: { id: '<id of memory>' } }, description: 'Remove specific short-term memory entry from the memory component.' },
+          // { example: { action: 'forgetLongTermMemory', where: { id: '<id of memory>' } }, description: 'Remove specific long-term memory entry from the memory component.' },
+          // { example: { action: 'searchGoogle', query: '<query>' }, description: 'Search Google with the query.' },
+        ]
+          .map((item, index) => `${index}: ${JSON.stringify(item.example)}: ${item.description}`)
+          .join('\n'),
+      ),
    ),
    message.user(''
      + 'What do you want to do? Respond with the action and parameters you choose in JSON only, without any explanation and markups',
@@ -0,0 +1,184 @@
+import type { Bot } from 'grammy'
+import type { Message, Sticker } from 'grammy/types'
+
+import { Buffer } from 'node:buffer'
+import { promises as fs } from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { env } from 'node:process'
+import ffmpegInstaller from '@ffmpeg-installer/ffmpeg'
+import { useLogg } from '@guiiai/logg'
+import { generateText } from '@xsai/generate-text'
+import { message } from '@xsai/utils-chat'
+import ffmpeg from 'fluent-ffmpeg'
+import Sharp from 'sharp'
+
+import { findStickerDescription, recordSticker } from '../models'
+import { span } from '../prompts/utils'
+
+// Set path to FFmpeg binaries
+ffmpeg.setFfmpegPath(ffmpegInstaller.path)
+
+async function extractFrames(inputFilePath, outputDir, frameRate = 5) {
+  await fs.mkdir(outputDir, { recursive: true })
+  const outputPattern = path.join(outputDir, 'frame-%03d.png')
+
+  return new Promise<string[]>((resolve, reject) => {
+    ffmpeg(inputFilePath)
+      .outputOptions(`-vf fps=${frameRate}`)
+      .output(outputPattern)
+      .on('end', async () => {
+        const files = await fs.readdir(outputDir)
+        const sortedFiles = files
+          .filter(file => file.match(/frame-\d+\.png/))
+          .sort((a, b) => {
+            const numA = Number.parseInt(a.match(/frame-(\d+)\.png/)[1])
+            const numB = Number.parseInt(b.match(/frame-(\d+)\.png/)[1])
+            return numA - numB
+          })
+        resolve(sortedFiles.map(file => path.join(outputDir, file)))
+      })
+      .on('error', err => reject(err))
+      .run()
+  })
+}
+
+export async function interpretAnimatedSticker(bot: Bot, msg: Message, sticker: Sticker) {
+  const logger = useLogg('interpretAnimatedSticker')
+    .useGlobalConfig()
+    .withField('sticker_id', sticker.file_id)
+    .withField('sticker_emoji', sticker.emoji)
+    .withField('sticker_set', sticker.set_name)
+
+  try {
+    if (await findStickerDescription(sticker.file_id)) {
+      logger.log('Sticker already interpreted, skipping')
+      return
+    }
+
+    const file = await bot.api.getFile(sticker.file_id)
+    const stickerRes = await fetch(`https://api.telegram.org/file/bot${bot.api.token}/${file.file_path}`)
+    const buffer = await stickerRes.arrayBuffer()
+
+    // Create temp directory for sticker and frames
+    const tempDir = path.join(os.tmpdir(), `sticker-${sticker.file_id}`)
+    const framesDir = path.join(tempDir, 'frames')
+    await fs.mkdir(tempDir, { recursive: true })
+    await fs.mkdir(framesDir, { recursive: true })
+
+    // Save sticker to temp file
+    const stickerPath = path.join(tempDir, path.basename(file.file_path))
+    await fs.writeFile(stickerPath, Buffer.from(buffer))
+
+    logger.withField('sticker_path', stickerPath).log('Sticker saved to temp file')
+
+    const framePaths = await extractFrames(stickerPath, framesDir, 15)
+    logger.withField('frame_paths', framePaths).log('Frames extracted')
+
+    // Sample frames if too many (limit to ~5-8 frames for processing)
+    let sampled = framePaths
+    if (framePaths.length > 8) {
+      const samplingRate = Math.ceil(framePaths.length / 8)
+      sampled = framePaths.filter((_, i) => i % samplingRate === 0)
+    }
+    logger.withField('sampled', sampled).log('Sampled frames')
+
+    // Process frames with Sharp
+    const frames = await Promise.all(sampled.map(async (framePath, index) => {
+      const frameBuffer = await Sharp(framePath)
+        .resize(512, 512)
+        .png()
+        .toBuffer()
+      return {
+        index,
+        buffer: frameBuffer,
+        base64: Buffer.from(frameBuffer).toString('base64'),
+      }
+    }))
+    logger.withField('sampled_frames', sampled).log('Normalized the frames')
+
+    // STAGE 1: Process each frame individually
+    const frameDescriptions = []
+    for (const frame of frames) {
+      try {
+        const res = await generateText({
+          apiKey: env.LLM_VISION_API_KEY!,
+          baseURL: env.LLM_VISION_API_BASE_URL!,
+          model: env.LLM_VISION_MODEL!,
+          messages: message.messages(
+            message.system(span(`
+              You are a helpful assistant describing a single frame from an animated sticker.
+              Focus only on describing what you see in this specific frame.
+              Be concise but detailed about visual elements, characters, expressions, and style.
+              This is frame ${frame.index + 1} in the sequence.
+            `)),
+            message.user([message.imagePart(`data:image/png;base64,${frame.base64}`)]),
+          ),
+        })
+
+        frameDescriptions.push({
+          frameNumber: frame.index + 1,
+          description: res.text,
+        })
+      }
+      catch (err) {
+        logger.withError(err).log(`Failed to process frame ${frame.index + 1}`)
+        // Continue with other frames
+      }
+    }
+
+    logger.withField('frames', frameDescriptions.length).log('Processed all frames')
+
+    // Only proceed to consolidation if we have at least some frames
+    if (frameDescriptions.length === 0) {
+      throw new Error('Failed to process any frames from the animated sticker')
+    }
+
+    // STAGE 2: Consolidate descriptions with a text-only LLM call
+    const consolidationPrompt = span(`
+      You are analyzing an animated sticker from Telegram.
+      The emoji associated with this sticker is: ${msg.sticker.emoji}
+      This sticker is from the set named: "${msg.sticker.set_name}"
+
+      Below are descriptions of ${frameDescriptions.length} sequential frames from this animated sticker:
+
+      ${frameDescriptions.map(fd => `FRAME ${fd.frameNumber}:\n${fd.description}\n`).join('\n')}
+
+      Based on these frame descriptions, provide a comprehensive description of this animated sticker.
+      Focus on:
+      1. What is being depicted overall
+      2. How the animation progresses (the movement/action)
+      3. The emotion or message the sticker is conveying
+      4. The visual style and notable characteristics
+      5. How this relates to the emoji ${msg.sticker.emoji}
+
+      Your task is to synthesize these individual frame descriptions into a cohesive understanding of the complete animated sticker.
+    `)
+
+    logger.log('Consolidating frames')
+
+    const consolidatedResult = await generateText({
+      apiKey: env.LLM_API_KEY!, // Using text-only LLM API
+      baseURL: env.LLM_API_BASE_URL!,
+      model: env.LLM_MODEL!,
+      messages: message.messages(
+        message.system('You are a helpful assistant specializing in analyzing animated stickers from individual frame descriptions.'),
+        message.user(consolidationPrompt),
+      ),
+    })
+
+    // Clean up temp files
+    await fs.rm(tempDir, { recursive: true, force: true })
+
+    logger.withField('consolidated_result', consolidatedResult.text).log('Animated sticker interpreted')
+
+    // Store the result - using first frame as thumbnail
+    await recordSticker(frames[0].base64, msg.sticker.file_id, file.file_path, consolidatedResult.text)
+    logger.withField('sticker', consolidatedResult.text).log('Interpreted animated sticker')
+
+    return consolidatedResult.text
+  }
+  catch (err) {
+    logger.withError(err).log('Error interpreting animated sticker')
+  }
+}
@@ -1,23 +1,38 @@
-import type { Message } from 'grammy/types'
-import type { BotSelf } from '../types'
+import type { Bot } from 'grammy'
+import type { Message, Sticker } from 'grammy/types'

 import { Buffer } from 'node:buffer'
 import { env } from 'node:process'
+import { useLogg } from '@guiiai/logg'
 import { embed } from '@xsai/embed'
 import { generateText } from '@xsai/generate-text'
 import { message } from '@xsai/utils-chat'
 import Sharp from 'sharp'

 import { findStickerDescription, recordSticker } from '../models'
+import { interpretAnimatedSticker } from './animated-sticker'
+
+export async function interpretSticker(bot: Bot, msg: Message, sticker: Sticker) {
+  const logger = useLogg('interpretSticker').useGlobalConfig()
+
+  if (sticker.is_animated || sticker.is_video) {
+    logger
+      .withField('sticker_emoji', sticker.emoji)
+      .withField('sticker_set', sticker.set_name)
+      .withField('is_animated', sticker.is_animated)
+      .withField('is_video', sticker.is_video)
+      .log('Animated or video sticker, interpreting as animated sticker')
+
+    return interpretAnimatedSticker(bot, msg, sticker)
+  }

-export async function interpretSticker(state: BotSelf, msg: Message) {
  try {
-    if (await findStickerDescription(msg.sticker.file_id)) {
+    if (await findStickerDescription(sticker.file_id)) {
      return
    }

-    const file = await state.bot.api.getFile(msg.sticker.file_id)
-    const stickerRes = await fetch(`https://api.telegram.org/file/bot${state.bot.api.token}/${file.file_path}`)
+    const file = await bot.api.getFile(sticker.file_id)
+    const stickerRes = await fetch(`https://api.telegram.org/file/bot${bot.api.token}/${file.file_path}`)
    const buffer = await stickerRes.arrayBuffer()
    const stickerBase64 = Buffer.from(await Sharp(buffer).resize(512, 512).png().toBuffer()).toString('base64')

@@ -56,9 +71,9 @@ export async function interpretSticker(state: BotSelf, msg: Message) {
    })

    await recordSticker(stickerBase64, msg.sticker.file_id, file.file_path, res.text)
-    state.logger.withField('sticker', res.text).log('Interpreted sticker')
+    logger.withField('sticker', res.text).log('Interpreted sticker')
  }
  catch (err) {
-    state.logger.withError(err).log('Error occurred')
+    logger.withError(err).log('Error occurred')
  }
 }
@@ -0,0 +1,13 @@
+import { useDrizzle } from '../db'
+import { stickerPacksTable } from '../db/schema'
+
+export async function recordStickerPack(platformId: string, name: string, platform = 'telegram') {
+  await useDrizzle()
+    .insert(stickerPacksTable)
+    .values({
+      platform,
+      platformId,
+      name,
+      description: '',
+    })
+}
@@ -140,27 +140,3 @@ export function systemPrompt() {
    + 'interpreted correctly.'
    + '')
 }
-
-export function memeAndFunnyChoicesGeneratorSystemPrompt() {
-  return message.system(''
-    + `${personality()}`
-    + 'Based on your personalities. Giving the above context and chat history, generate natural responses like a real Chinese netizen in the style of 2023 to 2025.'
-    + 'Rules:'
-    + '- NO 颜文字 (too old fashion)'
-    + '- Use trending slang from Bilibili/微博/知乎/小红书'
-    + '- Must be short (2-4 words max per message)'
-    + '- Don\'t use too many emojis'
-    + '- Can use English meme words mixed with Chinese'
-    + '\n'
-    + 'Examples of GOOD style:'
-    + '- ["草太真实了"]\n'
-    + '- ["?", "你搁这搁这呢"]\n'
-    + '- ["有道理", "就是说"]\n'
-    + '- ["kknd", "给个 link"]\n'
-    + '\n'
-    + 'Examples of BAD style (too old/formal):\n'
-    + '- ["(｀・ω・´)", "好的呢~"]\n'
-    + '- ["这个需求确实很有意思呢"]\n'
-    + '- ["让我想想哦..."]\n',
-  )
-}
@@ -87,7 +87,7 @@ export interface ListChatsAction {
 export interface SendMessageAction {
  action: 'sendMessage'
  content: string
-  groupId: string
+  chatId: string
 }

 export interface SearchGoogleAction {