From 3f91d8dab99c4d18acc81f663c7fb6f60e52ac4e Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Mon, 7 Apr 2025 18:28:22 +0800 Subject: [PATCH] fix(telegram-bot): field incorrect & sticker pack & animated stickers --- pnpm-lock.yaml | 106 ++ .../drizzle/0005_workable_shockwave.sql | 23 + .../drizzle/meta/0005_snapshot.json | 1505 +++++++++++++++++ .../telegram-bot/drizzle/meta/_journal.json | 7 + services/telegram-bot/package.json | 2 + .../src/bots/telegram/admin/sticker-packs.ts | 0 .../telegram-bot/src/bots/telegram/index.ts | 81 +- .../src/bots/telegram/loop/read-message.ts | 60 +- services/telegram-bot/src/db/schema.ts | 23 + services/telegram-bot/src/llm/actions.ts | 153 +- .../telegram-bot/src/llm/animated-sticker.ts | 184 ++ services/telegram-bot/src/llm/sticker.ts | 31 +- .../telegram-bot/src/memory/actor-memory.ts | 0 .../telegram-bot/src/models/sticker-packs.ts | 13 + .../telegram-bot/src/prompts/system-v1.ts | 24 - services/telegram-bot/src/types.ts | 2 +- 16 files changed, 2035 insertions(+), 179 deletions(-) create mode 100644 services/telegram-bot/drizzle/0005_workable_shockwave.sql create mode 100644 services/telegram-bot/drizzle/meta/0005_snapshot.json create mode 100644 services/telegram-bot/src/bots/telegram/admin/sticker-packs.ts create mode 100644 services/telegram-bot/src/llm/animated-sticker.ts create mode 100644 services/telegram-bot/src/memory/actor-memory.ts create mode 100644 services/telegram-bot/src/models/sticker-packs.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c766bc3cf..15942e9f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1255,6 +1255,9 @@ importers: '@dotenvx/dotenvx': specifier: ^1.39.1 version: 1.39.1 + '@ffmpeg-installer/ffmpeg': + specifier: ^1.1.0 + version: 1.1.0 '@grammyjs/files': specifier: ^1.1.1 version: 1.1.1(grammy@1.35.1(encoding@0.1.13)) @@ -1288,6 +1291,9 @@ importers: es-toolkit: specifier: ^1.34.1 version: 1.34.1 + fluent-ffmpeg: + specifier: ^2.1.3 + version: 2.1.3 grammy: specifier: ^1.35.1 version: 1.35.1(encoding@0.1.13) @@ -2995,6 +3001,49 @@ packages: '@expressive-code/plugin-text-markers@0.40.1': resolution: {integrity: sha512-LsirF7M4F2yWgrFXEocD74F/MaVXsOsHVsRxBLhXQJemSSkWkDp/EZPt//OaqQ8ExnqWZ2lH7E1/KiN46unKjg==} + '@ffmpeg-installer/darwin-arm64@4.1.5': + resolution: {integrity: sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==} + cpu: [arm64] + os: [darwin] + + '@ffmpeg-installer/darwin-x64@4.1.0': + resolution: {integrity: sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==} + cpu: [x64] + os: [darwin] + + '@ffmpeg-installer/ffmpeg@1.1.0': + resolution: {integrity: sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==} + + '@ffmpeg-installer/linux-arm64@4.1.4': + resolution: {integrity: sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==} + cpu: [arm64] + os: [linux] + + '@ffmpeg-installer/linux-arm@4.1.3': + resolution: {integrity: sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==} + cpu: [arm] + os: [linux] + + '@ffmpeg-installer/linux-ia32@4.1.0': + resolution: {integrity: sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==} + cpu: [ia32] + os: [linux] + + '@ffmpeg-installer/linux-x64@4.1.0': + resolution: {integrity: sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==} + cpu: [x64] + os: [linux] + + '@ffmpeg-installer/win32-ia32@4.1.0': + resolution: {integrity: sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==} + cpu: [ia32] + os: [win32] + + '@ffmpeg-installer/win32-x64@4.1.0': + resolution: {integrity: sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==} + cpu: [x64] + os: [win32] + '@floating-ui/core@1.6.8': resolution: {integrity: sha512-7XJ9cPU+yI2QeLS+FCSlqNFZJq8arvswefkZrYI1yQBbftw6FyrZOxYSh+9S7z7TpeWlRt9zJ5IhM1WIL334jA==} @@ -5760,6 +5809,9 @@ packages: async-mutex@0.3.2: resolution: {integrity: sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==} + async@0.2.10: + resolution: {integrity: sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==} + async@2.6.4: resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==} @@ -7316,6 +7368,10 @@ packages: resolution: {integrity: sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==} engines: {node: '>=8'} + fluent-ffmpeg@2.1.3: + resolution: {integrity: sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==} + engines: {node: '>=18'} + fluere@0.3.0-beta.1: resolution: {integrity: sha512-AY9+8ip/ImuC3ixLVc+Yq+YCneWcSkPYNJuRE6c/SAU9ZI8Pcdrhi1vPqJ55L0f8/t9OWbO2AOewT6tPnLAgMA==} peerDependencies: @@ -11478,6 +11534,10 @@ packages: resolution: {integrity: sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==} engines: {node: '>=4'} + which@1.3.1: + resolution: {integrity: sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==} + hasBin: true + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -13419,6 +13479,41 @@ snapshots: dependencies: '@expressive-code/core': 0.40.1 + '@ffmpeg-installer/darwin-arm64@4.1.5': + optional: true + + '@ffmpeg-installer/darwin-x64@4.1.0': + optional: true + + '@ffmpeg-installer/ffmpeg@1.1.0': + optionalDependencies: + '@ffmpeg-installer/darwin-arm64': 4.1.5 + '@ffmpeg-installer/darwin-x64': 4.1.0 + '@ffmpeg-installer/linux-arm': 4.1.3 + '@ffmpeg-installer/linux-arm64': 4.1.4 + '@ffmpeg-installer/linux-ia32': 4.1.0 + '@ffmpeg-installer/linux-x64': 4.1.0 + '@ffmpeg-installer/win32-ia32': 4.1.0 + '@ffmpeg-installer/win32-x64': 4.1.0 + + '@ffmpeg-installer/linux-arm64@4.1.4': + optional: true + + '@ffmpeg-installer/linux-arm@4.1.3': + optional: true + + '@ffmpeg-installer/linux-ia32@4.1.0': + optional: true + + '@ffmpeg-installer/linux-x64@4.1.0': + optional: true + + '@ffmpeg-installer/win32-ia32@4.1.0': + optional: true + + '@ffmpeg-installer/win32-x64@4.1.0': + optional: true + '@floating-ui/core@1.6.8': dependencies: '@floating-ui/utils': 0.2.9 @@ -16829,6 +16924,8 @@ snapshots: dependencies: tslib: 2.8.1 + async@0.2.10: {} + async@2.6.4: dependencies: lodash: 4.17.21 @@ -18704,6 +18801,11 @@ snapshots: flattie@1.1.1: {} + fluent-ffmpeg@2.1.3: + dependencies: + async: 0.2.10 + which: 1.3.1 + fluere@0.3.0-beta.1(@modelcontextprotocol/sdk@1.8.0)(zod@3.24.2): optionalDependencies: '@modelcontextprotocol/sdk': 1.8.0 @@ -24128,6 +24230,10 @@ snapshots: which-pm-runs@1.1.0: {} + which@1.3.1: + dependencies: + isexe: 2.0.0 + which@2.0.2: dependencies: isexe: 2.0.0 diff --git a/services/telegram-bot/drizzle/0005_workable_shockwave.sql b/services/telegram-bot/drizzle/0005_workable_shockwave.sql new file mode 100644 index 000000000..94d098688 --- /dev/null +++ b/services/telegram-bot/drizzle/0005_workable_shockwave.sql @@ -0,0 +1,23 @@ +CREATE TABLE "recent_sent_stickers" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "sticker_id" uuid NOT NULL, + "created_at" bigint DEFAULT 0 NOT NULL, + "updated_at" bigint DEFAULT 0 NOT NULL +); +--> statement-breakpoint +CREATE TABLE "sticker_packs" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "platform" text DEFAULT '' NOT NULL, + "platform_id" text DEFAULT '' NOT NULL, + "name" text DEFAULT '' NOT NULL, + "description" text DEFAULT '' NOT NULL, + "created_at" bigint DEFAULT 0 NOT NULL, + "updated_at" bigint DEFAULT 0 NOT NULL +); +--> statement-breakpoint +ALTER TABLE "photos" ADD COLUMN "caption" text DEFAULT '' NOT NULL;--> statement-breakpoint +ALTER TABLE "stickers" ADD COLUMN "name" text DEFAULT '' NOT NULL;--> statement-breakpoint +ALTER TABLE "stickers" ADD COLUMN "emoji" text DEFAULT '' NOT NULL;--> statement-breakpoint +ALTER TABLE "stickers" ADD COLUMN "label" text DEFAULT '' NOT NULL;--> statement-breakpoint +ALTER TABLE "recent_sent_stickers" ADD CONSTRAINT "recent_sent_stickers_sticker_id_stickers_id_fk" FOREIGN KEY ("sticker_id") REFERENCES "public"."stickers"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "sticker_packs_platform_platform_id_unique_index" ON "sticker_packs" USING btree ("platform","platform_id"); \ No newline at end of file diff --git a/services/telegram-bot/drizzle/meta/0005_snapshot.json b/services/telegram-bot/drizzle/meta/0005_snapshot.json new file mode 100644 index 000000000..222485de4 --- /dev/null +++ b/services/telegram-bot/drizzle/meta/0005_snapshot.json @@ -0,0 +1,1505 @@ +{ + "id": "286aea60-e837-4dbb-b2a2-f3636b19d3ff", + "prevId": "6099e3d7-80f3-4741-94ae-a274aeb80b42", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_completions_history": { + "name": "chat_completions_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "task": { + "name": "task", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "platform_message_id": { + "name": "platform_message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "from_id": { + "name": "from_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "from_name": { + "name": "from_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "in_chat_id": { + "name": "in_chat_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "is_reply": { + "name": "is_reply", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "reply_to_name": { + "name": "reply_to_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "reply_to_id": { + "name": "reply_to_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "content_vector_1536": { + "name": "content_vector_1536", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "content_vector_1024": { + "name": "content_vector_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "content_vector_768": { + "name": "content_vector_768", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_messages_content_vector_1536_index": { + "name": "chat_messages_content_vector_1536_index", + "columns": [ + { + "expression": "content_vector_1536", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "chat_messages_content_vector_1024_index": { + "name": "chat_messages_content_vector_1024_index", + "columns": [ + { + "expression": "content_vector_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "chat_messages_content_vector_768_index": { + "name": "chat_messages_content_vector_768_index", + "columns": [ + { + "expression": "content_vector_768", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.joined_chats": { + "name": "joined_chats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "chat_id": { + "name": "chat_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "chat_name": { + "name": "chat_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "joined_chats_chat_id_unique": { + "name": "joined_chats_chat_id_unique", + "nullsNotDistinct": false, + "columns": [ + "chat_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_episodic": { + "name": "memory_episodic", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "memory_id": { + "name": "memory_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "participants": { + "name": "participants", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_episodic_memory_id_index": { + "name": "memory_episodic_memory_id_index", + "columns": [ + { + "expression": "memory_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_episodic_event_type_index": { + "name": "memory_episodic_event_type_index", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_episodic_memory_id_memory_fragments_id_fk": { + "name": "memory_episodic_memory_id_memory_fragments_id_fk", + "tableFrom": "memory_episodic", + "tableTo": "memory_fragments", + "columnsFrom": [ + "memory_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_fragments": { + "name": "memory_fragments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "memory_type": { + "name": "memory_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "importance": { + "name": "importance", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "emotional_impact": { + "name": "emotional_impact", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_accessed": { + "name": "last_accessed", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "access_count": { + "name": "access_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "content_vector_1536": { + "name": "content_vector_1536", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "content_vector_1024": { + "name": "content_vector_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "content_vector_768": { + "name": "content_vector_768", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_items_content_vector_1536_index": { + "name": "memory_items_content_vector_1536_index", + "columns": [ + { + "expression": "content_vector_1536", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "memory_items_content_vector_1024_index": { + "name": "memory_items_content_vector_1024_index", + "columns": [ + { + "expression": "content_vector_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "memory_items_content_vector_768_index": { + "name": "memory_items_content_vector_768_index", + "columns": [ + { + "expression": "content_vector_768", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "memory_items_memory_type_index": { + "name": "memory_items_memory_type_index", + "columns": [ + { + "expression": "memory_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_items_category_index": { + "name": "memory_items_category_index", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_items_importance_index": { + "name": "memory_items_importance_index", + "columns": [ + { + "expression": "importance", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_items_created_at_index": { + "name": "memory_items_created_at_index", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_items_last_accessed_index": { + "name": "memory_items_last_accessed_index", + "columns": [ + { + "expression": "last_accessed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_long_term_goals": { + "name": "memory_long_term_goals", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "deadline": { + "name": "deadline", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": null + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'planned'" + }, + "parent_goal_id": { + "name": "parent_goal_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'personal'" + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_long_term_goals_priority_index": { + "name": "memory_long_term_goals_priority_index", + "columns": [ + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_long_term_goals_status_index": { + "name": "memory_long_term_goals_status_index", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_long_term_goals_deadline_index": { + "name": "memory_long_term_goals_deadline_index", + "columns": [ + { + "expression": "deadline", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_long_term_goals_parent_goal_id_index": { + "name": "memory_long_term_goals_parent_goal_id_index", + "columns": [ + { + "expression": "parent_goal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_long_term_goals_parent_goal_id_memory_long_term_goals_id_fk": { + "name": "memory_long_term_goals_parent_goal_id_memory_long_term_goals_id_fk", + "tableFrom": "memory_long_term_goals", + "tableTo": "memory_long_term_goals", + "columnsFrom": [ + "parent_goal_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_short_term_ideas": { + "name": "memory_short_term_ideas", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'dream'" + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": null + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'new'" + }, + "excitement": { + "name": "excitement", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "content_vector_1536": { + "name": "content_vector_1536", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "content_vector_1024": { + "name": "content_vector_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "content_vector_768": { + "name": "content_vector_768", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_short_term_ideas_source_type_index": { + "name": "memory_short_term_ideas_source_type_index", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_short_term_ideas_status_index": { + "name": "memory_short_term_ideas_status_index", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_short_term_ideas_excitement_index": { + "name": "memory_short_term_ideas_excitement_index", + "columns": [ + { + "expression": "excitement", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_short_term_ideas_content_vector_1536_index": { + "name": "memory_short_term_ideas_content_vector_1536_index", + "columns": [ + { + "expression": "content_vector_1536", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "memory_short_term_ideas_content_vector_1024_index": { + "name": "memory_short_term_ideas_content_vector_1024_index", + "columns": [ + { + "expression": "content_vector_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "memory_short_term_ideas_content_vector_768_index": { + "name": "memory_short_term_ideas_content_vector_768_index", + "columns": [ + { + "expression": "content_vector_768", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_tags": { + "name": "memory_tags", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "memory_id": { + "name": "memory_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_tags_memory_id_index": { + "name": "memory_tags_memory_id_index", + "columns": [ + { + "expression": "memory_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_tags_tag_index": { + "name": "memory_tags_tag_index", + "columns": [ + { + "expression": "tag", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_tags_memory_id_memory_fragments_id_fk": { + "name": "memory_tags_memory_id_memory_fragments_id_fk", + "tableFrom": "memory_tags", + "tableTo": "memory_fragments", + "columnsFrom": [ + "memory_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.photos": { + "name": "photos", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "image_base64": { + "name": "image_base64", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "image_path": { + "name": "image_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "caption": { + "name": "caption", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "description_vector_1536": { + "name": "description_vector_1536", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "description_vector_1024": { + "name": "description_vector_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "description_vector_768": { + "name": "description_vector_768", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "photos_description_vector_1536_index": { + "name": "photos_description_vector_1536_index", + "columns": [ + { + "expression": "description_vector_1536", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "photos_description_vector_1024_index": { + "name": "photos_description_vector_1024_index", + "columns": [ + { + "expression": "description_vector_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "photos_description_vector_768_index": { + "name": "photos_description_vector_768_index", + "columns": [ + { + "expression": "description_vector_768", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.recent_sent_stickers": { + "name": "recent_sent_stickers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "sticker_id": { + "name": "sticker_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "recent_sent_stickers_sticker_id_stickers_id_fk": { + "name": "recent_sent_stickers_sticker_id_stickers_id_fk", + "tableFrom": "recent_sent_stickers", + "tableTo": "stickers", + "columnsFrom": [ + "sticker_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sticker_packs": { + "name": "sticker_packs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "platform_id": { + "name": "platform_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "sticker_packs_platform_platform_id_unique_index": { + "name": "sticker_packs_platform_platform_id_unique_index", + "columns": [ + { + "expression": "platform", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "platform_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stickers": { + "name": "stickers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "emoji": { + "name": "emoji", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "file_id": { + "name": "file_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "image_base64": { + "name": "image_base64", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "image_path": { + "name": "image_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "description_vector_1536": { + "name": "description_vector_1536", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "description_vector_1024": { + "name": "description_vector_1024", + "type": "vector(1024)", + "primaryKey": false, + "notNull": false + }, + "description_vector_768": { + "name": "description_vector_768", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "stickers_description_vector_1536_index": { + "name": "stickers_description_vector_1536_index", + "columns": [ + { + "expression": "description_vector_1536", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "stickers_description_vector_1024_index": { + "name": "stickers_description_vector_1024_index", + "columns": [ + { + "expression": "description_vector_1024", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + }, + "stickers_description_vector_768_index": { + "name": "stickers_description_vector_768_index", + "columns": [ + { + "expression": "description_vector_768", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/services/telegram-bot/drizzle/meta/_journal.json b/services/telegram-bot/drizzle/meta/_journal.json index 3eb2fbe42..191c0ab7f 100644 --- a/services/telegram-bot/drizzle/meta/_journal.json +++ b/services/telegram-bot/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1743269104758, "tag": "0004_complete_gravity", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1744020688550, + "tag": "0005_workable_shockwave", + "breakpoints": true } ] } \ No newline at end of file diff --git a/services/telegram-bot/package.json b/services/telegram-bot/package.json index 66e70bff7..ff9fe6e71 100644 --- a/services/telegram-bot/package.json +++ b/services/telegram-bot/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "@dotenvx/dotenvx": "^1.39.1", + "@ffmpeg-installer/ffmpeg": "^1.1.0", "@grammyjs/files": "^1.1.1", "@guiiai/logg": "^1.0.7", "@xsai-ext/providers-cloud": "catalog:", @@ -33,6 +34,7 @@ "dotenv": "^16.4.7", "drizzle-orm": "^0.41.0", "es-toolkit": "^1.34.1", + "fluent-ffmpeg": "^2.1.3", "grammy": "^1.35.1", "nanoid": "^5.1.5", "p-limit": "^6.2.0", diff --git a/services/telegram-bot/src/bots/telegram/admin/sticker-packs.ts b/services/telegram-bot/src/bots/telegram/admin/sticker-packs.ts new file mode 100644 index 000000000..e69de29bb diff --git a/services/telegram-bot/src/bots/telegram/index.ts b/services/telegram-bot/src/bots/telegram/index.ts index 0cbda8313..353affa82 100644 --- a/services/telegram-bot/src/bots/telegram/index.ts +++ b/services/telegram-bot/src/bots/telegram/index.ts @@ -13,15 +13,11 @@ import { interpretPhotos } from '../../llm/photo' import { interpretSticker } from '../../llm/sticker' import { recordMessage } from '../../models' import { listJoinedChats, recordJoinedChat } from '../../models/chats' +import { recordStickerPack } from '../../models/sticker-packs' import { readMessage } from './loop/read-message' import { shouldInterruptProcessing } from './utils/interruption' import { sendMayStructuredMessage } from './utils/message' -async function isChatIdBotAdmin(chatId: number) { - const admins = env.ADMIN_USER_IDS!.split(',') - return admins.includes(chatId.toString()) -} - async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: string): Promise<() => Promise | undefined> { // Set the start time when beginning new processing state.currentProcessingStartTime = Date.now() @@ -56,6 +52,7 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri switch (action.action) { case 'readMessages': + { if (Object.keys(state.unreadMessages).length === 0) { state.logger.withField('action', action).log('No unread messages - deleting all unread messages') state.unreadMessages = {} @@ -66,7 +63,6 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri break } - // eslint-disable-next-line no-case-declarations let unreadMessagesForThisChat: Message[] | undefined = state.unreadMessages[action.chatId] // Modified interruption logic @@ -121,13 +117,21 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri // return { break: true } // } - await readMessage(state, state.bot.botInfo.id.toString(), chatId, action, unreadMessagesForThisChat, currentController) - return + const result = await readMessage(state, state.bot.botInfo.id.toString(), chatId, action, unreadMessagesForThisChat, currentController) + if (result?.result) { + msgs.push(message.user(`Reading message of chat ${action.chatId}:\n${result.result}`)) + return () => handleLoopStep(state, msgs, chatId) + } + else { + return + } + } case 'listChats': msgs.push(message.user(`List of chats:${(await listJoinedChats()).map(chat => `ID:${chat.chat_id}, Name:${chat.chat_name}`).join('\n')}`)) return () => handleLoopStep(state, msgs, chatId) case 'sendMessage': - await sendMayStructuredMessage(state, action.content, action.groupId) + msgs.push(message.user(`Sending message to group ${action.chatId}: ${action.content}`)) + await sendMayStructuredMessage(state, action.content, action.chatId) return default: msgs.push(message.user(`The action you sent ${action.action} haven't implemented yet by developer.`)) @@ -151,6 +155,19 @@ async function handleLoopStep(state: BotSelf, msgs?: LLMMessage[], chatId?: stri } } +async function isChatIdBotAdmin(fromId: number) { + if (!env.ADMIN_USER_IDS) { + return false + } + + const admins = env.ADMIN_USER_IDS.split(',') + if (admins.length === 0) { + return false + } + + return admins.includes(fromId.toString()) +} + async function handleLoop(state: BotSelf, msgs?: LLMMessage[], chatId?: string) { let result = await handleLoopStep(state, msgs, chatId) @@ -217,7 +234,7 @@ async function processMessageQueue(state: BotSelf) { if (nextMsg.status === 'pending') { if (nextMsg.message.sticker) { nextMsg.status = 'interpreting' - await interpretSticker(state, nextMsg.message) + await interpretSticker(state.bot, nextMsg.message, nextMsg.message.sticker) nextMsg.status = 'ready' } else if (nextMsg.message.photo) { @@ -283,10 +300,33 @@ export async function startTelegramBot() { const bot = new Bot(env.TELEGRAM_BOT_TOKEN!) const state = newBotSelf(bot, log) - bot.on('message:sticker', async (ctx) => { - if (ctx.message.sticker.is_animated || ctx.message.sticker.is_video) + bot.command('add_sticker_pack', async (ctx) => { + if (!(await isChatIdBotAdmin(ctx.message.from.id))) { + log.withField('from_id', ctx.message.from.id).log('not an admin - skipping') return + } + if (ctx.message.reply_to_message == null) { + await ctx.reply('Please reply to a sticker pack to add it.') + return + } + const logger = useLogg('addStickerPack').useGlobalConfig() + + const repliedSticker = ctx.message.reply_to_message.sticker + const stickerSet = await bot.api.getStickerSet(repliedSticker.set_name) + + logger.withField('sticker_set', repliedSticker.set_name).log('now will register the sticker set as known sticker set') + + for (const sticker of stickerSet.stickers) { + await interpretSticker(state.bot, ctx.message.reply_to_message, sticker) + logger.withField('sticker', sticker).log('interpreted sticker') + } + + await recordStickerPack(repliedSticker.set_name, stickerSet.name) + await ctx.reply('Sticker pack added.') + }) + + bot.on('message:sticker', async (ctx) => { const messageId = `${ctx.message.chat.id}-${ctx.message.message_id}` if (!state.processedIds.has(messageId)) { state.processedIds.add(messageId) @@ -325,24 +365,9 @@ export async function startTelegramBot() { processMessageQueue(state) }) - bot.command('load_sticker_pack', async (ctx) => { - if (!(await isChatIdBotAdmin(ctx.chat.id))) { - return - } - if (!ctx.message || !ctx.message.sticker) { - return - } - - await interpretSticker(state, ctx.message) - }) - - bot.errorHandler = async (err) => { - log.withError(err).log('Error occurred') - } - + bot.errorHandler = async err => log.withError(err).log('Error occurred') await bot.init() log.withField('bot_username', bot.botInfo.username).log('bot initialized') - bot.start({ drop_pending_updates: true }) try { diff --git a/services/telegram-bot/src/bots/telegram/loop/read-message.ts b/services/telegram-bot/src/bots/telegram/loop/read-message.ts index 33c8bc7ba..8a89a4f98 100644 --- a/services/telegram-bot/src/bots/telegram/loop/read-message.ts +++ b/services/telegram-bot/src/bots/telegram/loop/read-message.ts @@ -4,14 +4,9 @@ import type { BotSelf, ReadMessagesAction } from '../../../types' import { env } from 'node:process' import { useLogg } from '@guiiai/logg' import { embed } from '@xsai/embed' -import { generateText } from '@xsai/generate-text' -import { message } from '@xsai/utils-chat' import { findLastNMessages, findRelevantMessages } from '../../../models' -import { recordChatCompletions } from '../../../models/chat-completions-history' import { chatMessageToOneLine, telegramMessageToOneLine } from '../../../models/common' -import { personality } from '../../../prompts/system-v1' -import { sendMayStructuredMessage } from '../utils/message' export async function readMessage( state: BotSelf, @@ -23,6 +18,7 @@ export async function readMessage( ): Promise<{ loop?: boolean break?: boolean + result: string }> { const logger = useLogg('readMessage').useGlobalConfig() @@ -56,12 +52,9 @@ export async function readMessage( state.unreadMessages[action.chatId] = [] - const messages = message.messages( - personality(), - message.user('' - + `Currently, it\'s ${new Date()} on the server that hosts you.` - + 'The others in the group may live in a different timezone, so please be aware of the time difference.' - + '\n' + return { + break: true, + result: '' + 'You choose to read the messages from the group (perhaps you are already engaging the topics in the group).' + 'Imaging you are using Telegram app on the mobile phone, and you are reading the messages from the group chat.' + '\n' @@ -73,49 +66,6 @@ export async function readMessage( + '\n' + 'Relevant chat messages may help you recall the memories:\n' + `${relevantChatMessagesOneliner || 'No relevant messages'}` - + '\n' - + 'Based on your personalities, imaging you have your own choice and interest over different topics, ' - + 'giving the above context and chat history, would you like to participate in the conversation ' - + 'about the topic? Or will you aggressively diss or piss off about the opinions of others?\n' - + 'Feel free to ignore by just sending an empty array within a object with key "messages" (i.e.' - + '{ "messages": [] }).' - + 'If you would like to participate, send me an array of messages (i.e. { "messages": [] }) you would ' - + 'like to send without telling you willing to participate.' - + 'If you would like to reply to any of the message, send me an array of messages (i.e. { "messages":' - + '["message content"], "reply_to_message_id": "1234567890" }) with the message id of the message you ' - + 'want to reply to.' - + '\n' - + 'Choose your action.', - ), - ) - - let responseText = '' - - try { - const response = await generateText({ - apiKey: env.LLM_API_KEY!, - baseURL: env.LLM_API_BASE_URL!, - model: env.LLM_MODEL!, - messages, - abortSignal: abortController.signal, - }) - - responseText = response.text - .replace(/^```json\s*\n/, '') - .replace(/\n```$/, '') - .replace(/^```\s*\n/, '') - .replace(/\n```$/, '') - .trim() + + '\n', } - catch (err) { - logger.withField('error', err).log('Failed to generate response') - } - finally { - recordChatCompletions('readMessage', messages, responseText).then(() => {}).catch(err => logger.withField('error', err).log('Failed to record chat completions')) - } - - logger.withField('response', responseText).log('Successfully generated response') - await sendMayStructuredMessage(state, responseText, action.chatId.toString()) - - return { break: true } } diff --git a/services/telegram-bot/src/db/schema.ts b/services/telegram-bot/src/db/schema.ts index afa30e84a..a05e5232c 100644 --- a/services/telegram-bot/src/db/schema.ts +++ b/services/telegram-bot/src/db/schema.ts @@ -25,6 +25,9 @@ export const chatMessagesTable = pgTable('chat_messages', { export const stickersTable = pgTable('stickers', { id: uuid().primaryKey().defaultRandom(), platform: text().notNull().default(''), + name: text().notNull().default(''), + emoji: text().notNull().default(''), + label: text().notNull().default(''), file_id: text().notNull().default(''), image_base64: text().notNull().default(''), image_path: text().notNull().default(''), @@ -40,12 +43,32 @@ export const stickersTable = pgTable('stickers', { index('stickers_description_vector_768_index').using('hnsw', table.description_vector_768.op('vector_cosine_ops')), ]) +export const stickerPacksTable = pgTable('sticker_packs', { + id: uuid().primaryKey().defaultRandom(), + platform: text().notNull().default(''), + platform_id: text().notNull().default(''), + name: text().notNull().default(''), + description: text().notNull().default(''), + created_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()), + updated_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()), +}, table => [ + uniqueIndex('sticker_packs_platform_platform_id_unique_index').on(table.platform, table.platform_id), +]) + +export const recentSentStickersTable = pgTable('recent_sent_stickers', { + id: uuid().primaryKey().defaultRandom(), + sticker_id: uuid().notNull().references(() => stickersTable.id, { onDelete: 'cascade' }), + created_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()), + updated_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()), +}) + export const photosTable = pgTable('photos', { id: uuid().primaryKey().defaultRandom(), platform: text().notNull().default(''), file_id: text().notNull().default(''), image_base64: text().notNull().default(''), image_path: text().notNull().default(''), + caption: text().notNull().default(''), description: text().notNull().default(''), created_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()), updated_at: bigint({ mode: 'number' }).notNull().default(0).$defaultFn(() => Date.now()), diff --git a/services/telegram-bot/src/llm/actions.ts b/services/telegram-bot/src/llm/actions.ts index 165cc42f0..f6c60f40d 100644 --- a/services/telegram-bot/src/llm/actions.ts +++ b/services/telegram-bot/src/llm/actions.ts @@ -10,6 +10,7 @@ import { parse } from 'best-effort-json-parser' import { recordChatCompletions } from '../models/chat-completions-history' import { personality } from '../prompts/system-v1' +import { div, span } from '../prompts/utils' export async function imagineAnAction( _botId: string, @@ -25,70 +26,96 @@ export async function imagineAnAction( } agentMessages.push( - message.system('' - + `${personality().content}` - + '\n' - + 'I am one of your system component, called Ticking system, which is responsible to keep track of the time, and ' - + 'help you schedule, retain focus, and keep eyes on different tasks, and ideas you have.' - + '\n' - + 'Please remember, I am not a human, I am a system that part of you. You should return system/machine readable ' - + 'messages to me, so I can understand and process them correctly.' - + '\n' - + 'Now, please, based on the following context, choose a right action from the listing of the tools you want to ' - + 'take next:', - ), message.system( - [ - { - description: 'List all available chats, best to do before you want to send a message to a chat.', - example: { action: 'listChats' }, - }, - { - description: `Send a message to a specific chat group. If you want to express anything to anyone or your friends in group, you can use this action.${!!env.LLM_RESPONSE_LANGUAGE}` ? `The language of the sending message should be in ${env.LLM_RESPONSE_LANGUAGE}.` : '', - example: { action: 'sendMessage', content: '', chatId: 'id of chat to send to', reply_to_message_id: 'id of message to reply to (Optional)' }, - }, - { - description: 'Read unread messages from a specific chat group. If you want to read the unread messages from a specific chat group, you can use this action.', - example: { action: 'readMessages', chatId: 'id of chat to send to' }, - }, - { - description: 'Continue the current task, which means to keep your current state unchanged, I\'ll ask you again in next tick.', - example: { action: 'continue' }, - }, - { - description: 'Take a break, which means to clear out ongoing tasks, but keep the short-term memory, and I\'ll ask you again in next tick.', - example: { action: 'break' }, - }, - { - description: 'Sleep, which means to clear out ongoing tasks, and clear out the working memory, and I\'ll ask you again in next tick.', - example: { action: 'sleep' }, - }, - { - description: 'By giving references to contexts, come up ideas to record in long-term memory.', - example: { action: 'comeUpIdeas', ideas: ['I want to tell everyone a story of myself', 'I want to google how to make a AI like me'] }, - }, - { - description: 'By giving references to contexts, come up goals with deadline and priority to record in long-term memory.', - example: { action: 'comeUpGoals', goals: [{ text: 'Learn to play Minecraft', deadline: '2025-05-01 23:59:59', priority: 6 }, { text: 'Learn anime of this season', deadline: '2025-01-08 23:59:59', priority: 9 }] }, - }, - // { example: { action: 'lookupShortTermMemory', query: '', category: 'chat or self' }, description: 'Look up the short-term, which means to recall the short-term memory from memory component.' }, - // { example: { action: 'lookupLongTermMemory', query: '', category: 'chat or self' }, description: 'Look up the long-term, which means to recall the long-term memory from memory component.' }, - // { example: { action: 'memorizeShortMemory', content: '', tags: ['keyword tag'] }, description: 'Memorize to short-term memory, which means to append things the short-term memory which will be included for a while, but will be eventually forgot.' }, - // { example: { action: 'memorizeLongMemory', content: '', tags: ['keyword tag'] }, description: 'Memorize to long-term memory, which means to append things the long-term memory which will be included for a long time, and hard to forget.' }, - // { example: { action: 'forgetShortTermMemory', where: { id: '' } }, description: 'Remove specific short-term memory entry from the memory component.' }, - // { example: { action: 'forgetLongTermMemory', where: { id: '' } }, description: 'Remove specific long-term memory entry from the memory component.' }, - // { example: { action: 'searchGoogle', query: '' }, description: 'Search Google with the query.' }, - ] - .map((item, index) => `${index}: ${JSON.stringify(item.example)}: ${item.description}`) - .join('\n'), - ), - message.system('' - + `Now the time is: ${new Date().toLocaleString()}. ` - + `You have total ${Object.values(unreadMessages).reduce((acc, cur) => acc + cur.length, 0)} unread messages.` - + '\n' - + 'Unread messages count are:\n' - + `${Object.entries(unreadMessages).map(([key, value]) => `ID:${key}, Unread message count:${value.length}`).join('\n')}` - + '', + div( + personality(), + span(` + I am one of your system component, called Ticking system, which is responsible to keep track of the time, and + help you schedule, retain focus, and keep eyes on different tasks, and ideas you have. + `), + span(` + Please remember, I am not a human, I am a system that part of you. You should return system/machine readable + messages to me, so I can understand and process them correctly. + `), + span(` + Currently, it's ${new Date()} on the server that hosts you. + The others in the group may live in a different timezone, so please be aware of the time difference. + `), + span(` + Based on your personalities, imaging you have your own choice and interest over different topics, + giving the above context and chat history, would you like to participate in the conversation + about the topic? Or will you aggressively diss or piss off about the opinions of others?\n + Feel free to ignore by just sending an empty array within a object with key "messages" (i.e. + { "messages": [] }). + `), + span(` + If you would like to participate, send me an array of messages (i.e. { "messages": [] }) you would + like to send without telling you willing to participate. + `), + span(` + If you would like to reply to any of the message, send me an array of messages (i.e. { "messages": + ["message content"], "reply_to_message_id": "1234567890" }) with the message id of the message you + want to reply to. + `), + span(` + You have total ${Object.values(unreadMessages).reduce((acc, cur) => acc + cur.length, 0)} unread messages. + `), + div( + 'Unread messages count are:', + ...Object.entries(unreadMessages).map(([key, value]) => `ID:${key}, Unread message count:${value.length}`).join('\n'), + ), + span(` + Now, please, based on the context, choose a right action from the listing of the tools you want to + take next: + `), + [ + { + description: 'List all available chats, best to do before you want to send a message to a chat.', + example: { action: 'listChats' }, + }, + { + description: '' + + 'Send a message to a specific chat group.If you want to express anything to anyone or your friends' + + 'in group, you can use this action.' + + 'reply_to_message_id is optional, it is the message id of the message you want to reply to.' + + `${env.LLM_RESPONSE_LANGUAGE ? `The language of the sending message should be in ${env.LLM_RESPONSE_LANGUAGE}.` : ''}`, + example: { action: 'sendMessage', content: '', chatId: '-1001231231234', reply_to_message_id: '151' }, + }, + { + description: 'Read unread messages from a specific chat group. If you want to read the unread messages from a specific chat group, you can use this action.', + example: { action: 'readMessages', chatId: '-1001231231234' }, + }, + { + description: 'Continue the current task, which means to keep your current state unchanged, I\'ll ask you again in next tick.', + example: { action: 'continue' }, + }, + { + description: 'Take a break, which means to clear out ongoing tasks, but keep the short-term memory, and I\'ll ask you again in next tick.', + example: { action: 'break' }, + }, + { + description: 'Sleep, which means to clear out ongoing tasks, and clear out the working memory, and I\'ll ask you again in next tick.', + example: { action: 'sleep' }, + }, + { + description: 'By giving references to contexts, come up ideas to record in long-term memory.', + example: { action: 'comeUpIdeas', ideas: ['I want to tell everyone a story of myself', 'I want to google how to make a AI like me'] }, + }, + { + description: 'By giving references to contexts, come up goals with deadline and priority to record in long-term memory.', + example: { action: 'comeUpGoals', goals: [{ text: 'Learn to play Minecraft', deadline: '2025-05-01 23:59:59', priority: 6 }, { text: 'Learn anime of this season', deadline: '2025-01-08 23:59:59', priority: 9 }] }, + }, + // { example: { action: 'lookupShortTermMemory', query: '', category: 'chat or self' }, description: 'Look up the short-term, which means to recall the short-term memory from memory component.' }, + // { example: { action: 'lookupLongTermMemory', query: '', category: 'chat or self' }, description: 'Look up the long-term, which means to recall the long-term memory from memory component.' }, + // { example: { action: 'memorizeShortMemory', content: '', tags: ['keyword tag'] }, description: 'Memorize to short-term memory, which means to append things the short-term memory which will be included for a while, but will be eventually forgot.' }, + // { example: { action: 'memorizeLongMemory', content: '', tags: ['keyword tag'] }, description: 'Memorize to long-term memory, which means to append things the long-term memory which will be included for a long time, and hard to forget.' }, + // { example: { action: 'forgetShortTermMemory', where: { id: '' } }, description: 'Remove specific short-term memory entry from the memory component.' }, + // { example: { action: 'forgetLongTermMemory', where: { id: '' } }, description: 'Remove specific long-term memory entry from the memory component.' }, + // { example: { action: 'searchGoogle', query: '' }, description: 'Search Google with the query.' }, + ] + .map((item, index) => `${index}: ${JSON.stringify(item.example)}: ${item.description}`) + .join('\n'), + ), ), message.user('' + 'What do you want to do? Respond with the action and parameters you choose in JSON only, without any explanation and markups', diff --git a/services/telegram-bot/src/llm/animated-sticker.ts b/services/telegram-bot/src/llm/animated-sticker.ts new file mode 100644 index 000000000..59c5226a3 --- /dev/null +++ b/services/telegram-bot/src/llm/animated-sticker.ts @@ -0,0 +1,184 @@ +import type { Bot } from 'grammy' +import type { Message, Sticker } from 'grammy/types' + +import { Buffer } from 'node:buffer' +import { promises as fs } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { env } from 'node:process' +import ffmpegInstaller from '@ffmpeg-installer/ffmpeg' +import { useLogg } from '@guiiai/logg' +import { generateText } from '@xsai/generate-text' +import { message } from '@xsai/utils-chat' +import ffmpeg from 'fluent-ffmpeg' +import Sharp from 'sharp' + +import { findStickerDescription, recordSticker } from '../models' +import { span } from '../prompts/utils' + +// Set path to FFmpeg binaries +ffmpeg.setFfmpegPath(ffmpegInstaller.path) + +async function extractFrames(inputFilePath, outputDir, frameRate = 5) { + await fs.mkdir(outputDir, { recursive: true }) + const outputPattern = path.join(outputDir, 'frame-%03d.png') + + return new Promise((resolve, reject) => { + ffmpeg(inputFilePath) + .outputOptions(`-vf fps=${frameRate}`) + .output(outputPattern) + .on('end', async () => { + const files = await fs.readdir(outputDir) + const sortedFiles = files + .filter(file => file.match(/frame-\d+\.png/)) + .sort((a, b) => { + const numA = Number.parseInt(a.match(/frame-(\d+)\.png/)[1]) + const numB = Number.parseInt(b.match(/frame-(\d+)\.png/)[1]) + return numA - numB + }) + resolve(sortedFiles.map(file => path.join(outputDir, file))) + }) + .on('error', err => reject(err)) + .run() + }) +} + +export async function interpretAnimatedSticker(bot: Bot, msg: Message, sticker: Sticker) { + const logger = useLogg('interpretAnimatedSticker') + .useGlobalConfig() + .withField('sticker_id', sticker.file_id) + .withField('sticker_emoji', sticker.emoji) + .withField('sticker_set', sticker.set_name) + + try { + if (await findStickerDescription(sticker.file_id)) { + logger.log('Sticker already interpreted, skipping') + return + } + + const file = await bot.api.getFile(sticker.file_id) + const stickerRes = await fetch(`https://api.telegram.org/file/bot${bot.api.token}/${file.file_path}`) + const buffer = await stickerRes.arrayBuffer() + + // Create temp directory for sticker and frames + const tempDir = path.join(os.tmpdir(), `sticker-${sticker.file_id}`) + const framesDir = path.join(tempDir, 'frames') + await fs.mkdir(tempDir, { recursive: true }) + await fs.mkdir(framesDir, { recursive: true }) + + // Save sticker to temp file + const stickerPath = path.join(tempDir, path.basename(file.file_path)) + await fs.writeFile(stickerPath, Buffer.from(buffer)) + + logger.withField('sticker_path', stickerPath).log('Sticker saved to temp file') + + const framePaths = await extractFrames(stickerPath, framesDir, 15) + logger.withField('frame_paths', framePaths).log('Frames extracted') + + // Sample frames if too many (limit to ~5-8 frames for processing) + let sampled = framePaths + if (framePaths.length > 8) { + const samplingRate = Math.ceil(framePaths.length / 8) + sampled = framePaths.filter((_, i) => i % samplingRate === 0) + } + logger.withField('sampled', sampled).log('Sampled frames') + + // Process frames with Sharp + const frames = await Promise.all(sampled.map(async (framePath, index) => { + const frameBuffer = await Sharp(framePath) + .resize(512, 512) + .png() + .toBuffer() + return { + index, + buffer: frameBuffer, + base64: Buffer.from(frameBuffer).toString('base64'), + } + })) + logger.withField('sampled_frames', sampled).log('Normalized the frames') + + // STAGE 1: Process each frame individually + const frameDescriptions = [] + for (const frame of frames) { + try { + const res = await generateText({ + apiKey: env.LLM_VISION_API_KEY!, + baseURL: env.LLM_VISION_API_BASE_URL!, + model: env.LLM_VISION_MODEL!, + messages: message.messages( + message.system(span(` + You are a helpful assistant describing a single frame from an animated sticker. + Focus only on describing what you see in this specific frame. + Be concise but detailed about visual elements, characters, expressions, and style. + This is frame ${frame.index + 1} in the sequence. + `)), + message.user([message.imagePart(`data:image/png;base64,${frame.base64}`)]), + ), + }) + + frameDescriptions.push({ + frameNumber: frame.index + 1, + description: res.text, + }) + } + catch (err) { + logger.withError(err).log(`Failed to process frame ${frame.index + 1}`) + // Continue with other frames + } + } + + logger.withField('frames', frameDescriptions.length).log('Processed all frames') + + // Only proceed to consolidation if we have at least some frames + if (frameDescriptions.length === 0) { + throw new Error('Failed to process any frames from the animated sticker') + } + + // STAGE 2: Consolidate descriptions with a text-only LLM call + const consolidationPrompt = span(` + You are analyzing an animated sticker from Telegram. + The emoji associated with this sticker is: ${msg.sticker.emoji} + This sticker is from the set named: "${msg.sticker.set_name}" + + Below are descriptions of ${frameDescriptions.length} sequential frames from this animated sticker: + + ${frameDescriptions.map(fd => `FRAME ${fd.frameNumber}:\n${fd.description}\n`).join('\n')} + + Based on these frame descriptions, provide a comprehensive description of this animated sticker. + Focus on: + 1. What is being depicted overall + 2. How the animation progresses (the movement/action) + 3. The emotion or message the sticker is conveying + 4. The visual style and notable characteristics + 5. How this relates to the emoji ${msg.sticker.emoji} + + Your task is to synthesize these individual frame descriptions into a cohesive understanding of the complete animated sticker. + `) + + logger.log('Consolidating frames') + + const consolidatedResult = await generateText({ + apiKey: env.LLM_API_KEY!, // Using text-only LLM API + baseURL: env.LLM_API_BASE_URL!, + model: env.LLM_MODEL!, + messages: message.messages( + message.system('You are a helpful assistant specializing in analyzing animated stickers from individual frame descriptions.'), + message.user(consolidationPrompt), + ), + }) + + // Clean up temp files + await fs.rm(tempDir, { recursive: true, force: true }) + + logger.withField('consolidated_result', consolidatedResult.text).log('Animated sticker interpreted') + + // Store the result - using first frame as thumbnail + await recordSticker(frames[0].base64, msg.sticker.file_id, file.file_path, consolidatedResult.text) + logger.withField('sticker', consolidatedResult.text).log('Interpreted animated sticker') + + return consolidatedResult.text + } + catch (err) { + logger.withError(err).log('Error interpreting animated sticker') + } +} diff --git a/services/telegram-bot/src/llm/sticker.ts b/services/telegram-bot/src/llm/sticker.ts index 112cd1b73..b19a9ab81 100644 --- a/services/telegram-bot/src/llm/sticker.ts +++ b/services/telegram-bot/src/llm/sticker.ts @@ -1,23 +1,38 @@ -import type { Message } from 'grammy/types' -import type { BotSelf } from '../types' +import type { Bot } from 'grammy' +import type { Message, Sticker } from 'grammy/types' import { Buffer } from 'node:buffer' import { env } from 'node:process' +import { useLogg } from '@guiiai/logg' import { embed } from '@xsai/embed' import { generateText } from '@xsai/generate-text' import { message } from '@xsai/utils-chat' import Sharp from 'sharp' import { findStickerDescription, recordSticker } from '../models' +import { interpretAnimatedSticker } from './animated-sticker' + +export async function interpretSticker(bot: Bot, msg: Message, sticker: Sticker) { + const logger = useLogg('interpretSticker').useGlobalConfig() + + if (sticker.is_animated || sticker.is_video) { + logger + .withField('sticker_emoji', sticker.emoji) + .withField('sticker_set', sticker.set_name) + .withField('is_animated', sticker.is_animated) + .withField('is_video', sticker.is_video) + .log('Animated or video sticker, interpreting as animated sticker') + + return interpretAnimatedSticker(bot, msg, sticker) + } -export async function interpretSticker(state: BotSelf, msg: Message) { try { - if (await findStickerDescription(msg.sticker.file_id)) { + if (await findStickerDescription(sticker.file_id)) { return } - const file = await state.bot.api.getFile(msg.sticker.file_id) - const stickerRes = await fetch(`https://api.telegram.org/file/bot${state.bot.api.token}/${file.file_path}`) + const file = await bot.api.getFile(sticker.file_id) + const stickerRes = await fetch(`https://api.telegram.org/file/bot${bot.api.token}/${file.file_path}`) const buffer = await stickerRes.arrayBuffer() const stickerBase64 = Buffer.from(await Sharp(buffer).resize(512, 512).png().toBuffer()).toString('base64') @@ -56,9 +71,9 @@ export async function interpretSticker(state: BotSelf, msg: Message) { }) await recordSticker(stickerBase64, msg.sticker.file_id, file.file_path, res.text) - state.logger.withField('sticker', res.text).log('Interpreted sticker') + logger.withField('sticker', res.text).log('Interpreted sticker') } catch (err) { - state.logger.withError(err).log('Error occurred') + logger.withError(err).log('Error occurred') } } diff --git a/services/telegram-bot/src/memory/actor-memory.ts b/services/telegram-bot/src/memory/actor-memory.ts new file mode 100644 index 000000000..e69de29bb diff --git a/services/telegram-bot/src/models/sticker-packs.ts b/services/telegram-bot/src/models/sticker-packs.ts new file mode 100644 index 000000000..35667875b --- /dev/null +++ b/services/telegram-bot/src/models/sticker-packs.ts @@ -0,0 +1,13 @@ +import { useDrizzle } from '../db' +import { stickerPacksTable } from '../db/schema' + +export async function recordStickerPack(platformId: string, name: string, platform = 'telegram') { + await useDrizzle() + .insert(stickerPacksTable) + .values({ + platform, + platformId, + name, + description: '', + }) +} diff --git a/services/telegram-bot/src/prompts/system-v1.ts b/services/telegram-bot/src/prompts/system-v1.ts index b55178d52..8b16d1aff 100644 --- a/services/telegram-bot/src/prompts/system-v1.ts +++ b/services/telegram-bot/src/prompts/system-v1.ts @@ -140,27 +140,3 @@ export function systemPrompt() { + 'interpreted correctly.' + '') } - -export function memeAndFunnyChoicesGeneratorSystemPrompt() { - return message.system('' - + `${personality()}` - + 'Based on your personalities. Giving the above context and chat history, generate natural responses like a real Chinese netizen in the style of 2023 to 2025.' - + 'Rules:' - + '- NO 颜文字 (too old fashion)' - + '- Use trending slang from Bilibili/微博/知乎/小红书' - + '- Must be short (2-4 words max per message)' - + '- Don\'t use too many emojis' - + '- Can use English meme words mixed with Chinese' - + '\n' - + 'Examples of GOOD style:' - + '- ["草太真实了"]\n' - + '- ["?", "你搁这搁这呢"]\n' - + '- ["有道理", "就是说"]\n' - + '- ["kknd", "给个 link"]\n' - + '\n' - + 'Examples of BAD style (too old/formal):\n' - + '- ["(｀・ω・´)", "好的呢~"]\n' - + '- ["这个需求确实很有意思呢"]\n' - + '- ["让我想想哦..."]\n', - ) -} diff --git a/services/telegram-bot/src/types.ts b/services/telegram-bot/src/types.ts index 96b86a937..c185f068b 100644 --- a/services/telegram-bot/src/types.ts +++ b/services/telegram-bot/src/types.ts @@ -87,7 +87,7 @@ export interface ListChatsAction { export interface SendMessageAction { action: 'sendMessage' content: string - groupId: string + chatId: string } export interface SearchGoogleAction {