feat(docs):Added dev logs and improved navigation (#713)
This commit is contained in:
Lovehsigure_520
@@ -26,6 +26,12 @@ const showFooter = computed(
|
||||
|| control.value.prev
|
||||
|| control.value.next,
|
||||
)
|
||||
|
||||
/**
|
||||
* Footer navigation visibility rules (English-only comments as requested):
|
||||
* - Render nav only when prev/next have actual links.
|
||||
* - Do NOT show a disabled next button on the last article.
|
||||
*/
|
||||
</script>
|
||||
|
||||
<template>
|
||||
|
||||
@@ -1,17 +1,130 @@
|
||||
import { useData, withBase } from 'vitepress'
|
||||
import { computed } from 'vue'
|
||||
|
||||
import { data as blogPosts } from '../functions/blog.data'
|
||||
import { getFlatSideBarLinks, getSidebar, isActive } from './sidebar'
|
||||
|
||||
/**
|
||||
* Compute previous/next navigation targets for the current page.
|
||||
* - For blog pages, keeps navigation within the same language blog directory.
|
||||
* - For docs pages, falls back to sidebar-based navigation.
|
||||
* - Respects frontmatter overrides and hides when disabled.
|
||||
*/
|
||||
export function usePrevNext() {
|
||||
const { page, theme, frontmatter } = useData()
|
||||
const { page, theme, frontmatter, lang } = useData()
|
||||
|
||||
return computed(() => {
|
||||
// Blog-specific navigation: ensure next/prev stay within same language blog directory
|
||||
// This handles the case where clicking the next button on a blog post should navigate
|
||||
// to the next post within `/zh-Hans/blog/` or `/en/blog/`, preserving language prefix.
|
||||
const isBlogPage = page.value.relativePath.includes('/blog/')
|
||||
|
||||
// Determine visibility from theme/frontmatter first
|
||||
const hidePrev
|
||||
= (theme.value.docFooter?.prev === false && !frontmatter.value.prev)
|
||||
|| frontmatter.value.prev === false
|
||||
|
||||
const hideNext
|
||||
= (theme.value.docFooter?.next === false && !frontmatter.value.next)
|
||||
|| frontmatter.value.next === false
|
||||
|
||||
if (isBlogPage) {
|
||||
// Filter posts by current language and exclude blog index page
|
||||
const sameLangPosts = blogPosts
|
||||
.filter(p => p.lang === (lang.value || 'en'))
|
||||
.filter(p => p.urlWithoutLang !== '/blog/')
|
||||
|
||||
// Find current post index by matching normalized URLs
|
||||
let currentPath = page.value.relativePath
|
||||
if (currentPath.startsWith('/')) {
|
||||
currentPath = currentPath.slice(1)
|
||||
}
|
||||
|
||||
const currentUrl = withBase(`/${currentPath}`)
|
||||
const currentIndex = sameLangPosts.findIndex(p => isActive(currentUrl, withBase(p.url)))
|
||||
|
||||
// Gracefully handle not found index
|
||||
const prevPost = currentIndex > 0 ? sameLangPosts[currentIndex - 1] : undefined
|
||||
const nextPost = currentIndex >= 0 && currentIndex < sameLangPosts.length - 1 ? sameLangPosts[currentIndex + 1] : undefined
|
||||
|
||||
// For blog pages, do NOT render next when it's the last article,
|
||||
// even if frontmatter provides a manual next link.
|
||||
const blogPrev = hidePrev
|
||||
? undefined
|
||||
: prevPost
|
||||
? {
|
||||
text:
|
||||
(typeof frontmatter.value.prev === 'string'
|
||||
? frontmatter.value.prev
|
||||
: typeof frontmatter.value.prev === 'object'
|
||||
? frontmatter.value.prev.text
|
||||
: undefined)
|
||||
?? prevPost.title,
|
||||
link: withBase(prevPost.url),
|
||||
}
|
||||
: undefined
|
||||
|
||||
const blogNext = hideNext
|
||||
? undefined
|
||||
: nextPost
|
||||
? {
|
||||
text:
|
||||
(typeof frontmatter.value.next === 'string'
|
||||
? frontmatter.value.next
|
||||
: typeof frontmatter.value.next === 'object'
|
||||
? frontmatter.value.next.text
|
||||
: undefined)
|
||||
?? nextPost.title,
|
||||
link: withBase(nextPost.url),
|
||||
}
|
||||
: undefined
|
||||
|
||||
return {
|
||||
prev: blogPrev,
|
||||
next: blogNext,
|
||||
} as {
|
||||
prev?: { text?: string, link?: string }
|
||||
next?: { text?: string, link?: string }
|
||||
}
|
||||
}
|
||||
|
||||
// Default docs navigation via sidebar for non-blog pages
|
||||
const sidebar = getSidebar(theme.value.sidebar, page.value.relativePath)
|
||||
const links = getFlatSideBarLinks(sidebar)
|
||||
|
||||
// ignore inner-page links with hashes
|
||||
const candidates = uniqBy(links, link => link.link.replace(/[?#].*$/, ''))
|
||||
let candidates = uniqBy(links, link => link.link.replace(/[?#].*$/, ''))
|
||||
|
||||
// Restrict docs navigation within the same docs section (e.g., overview vs manual)
|
||||
// This prevents crossing into unrelated sections like `/zh-Hans/docs/manual/`.
|
||||
let normalizedPath = page.value.relativePath
|
||||
if (normalizedPath.startsWith('/')) {
|
||||
normalizedPath = normalizedPath.slice(1)
|
||||
}
|
||||
const currentFullUrl = withBase(`/${normalizedPath}`)
|
||||
const sectionPrefix = getDocsSectionPrefix(currentFullUrl)
|
||||
if (sectionPrefix) {
|
||||
const sectionBase = withBase(sectionPrefix)
|
||||
// If current page is the section root (e.g., /zh-Hans/docs/overview/),
|
||||
// do not render prev/next for docs to avoid showing a next button here.
|
||||
const isSectionRoot = currentFullUrl.replace(/[?#].*$/, '') === sectionBase
|
||||
if (isSectionRoot) {
|
||||
return {
|
||||
prev: undefined,
|
||||
next: undefined,
|
||||
} as { prev?: { text?: string, link?: string }, next?: { text?: string, link?: string } }
|
||||
}
|
||||
// Keep navigation within the same docs section and exclude the section root itself
|
||||
// to avoid showing a "next" link that points back to the section index.
|
||||
const filtered = candidates
|
||||
.filter(l => l.link.replace(/[?#].*$/, '').startsWith(sectionBase))
|
||||
.filter(l => l.link.replace(/[?#].*$/, '') !== sectionBase)
|
||||
// Fallback to all candidates if filter would drop the current page
|
||||
const wouldDropCurrent = filtered.findIndex(l => isActive(currentFullUrl, l.link)) < 0
|
||||
if (!wouldDropCurrent) {
|
||||
candidates = filtered
|
||||
}
|
||||
}
|
||||
|
||||
const index = candidates.findIndex((link) => {
|
||||
let path = page.value.relativePath
|
||||
@@ -22,16 +135,8 @@ export function usePrevNext() {
|
||||
return isActive(withBase(`/${path}`), link.link)
|
||||
})
|
||||
|
||||
const hidePrev
|
||||
= (theme.value.docFooter?.prev === false && !frontmatter.value.prev)
|
||||
|| frontmatter.value.prev === false
|
||||
|
||||
const hideNext
|
||||
= (theme.value.docFooter?.next === false && !frontmatter.value.next)
|
||||
|| frontmatter.value.next === false
|
||||
|
||||
return {
|
||||
prev: hidePrev
|
||||
prev: hidePrev || index <= 0
|
||||
? undefined
|
||||
: {
|
||||
text:
|
||||
@@ -47,7 +152,7 @@ export function usePrevNext() {
|
||||
? frontmatter.value.prev.link
|
||||
: undefined) ?? candidates[index - 1]?.link,
|
||||
},
|
||||
next: hideNext
|
||||
next: hideNext || index < 0 || index >= candidates.length - 1
|
||||
? undefined
|
||||
: {
|
||||
text:
|
||||
@@ -70,6 +175,19 @@ export function usePrevNext() {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract docs section prefix like `/<lang>/docs/<section>/` from a full URL.
|
||||
* Keeps navigation within the same section to avoid crossing into unrelated docs.
|
||||
*/
|
||||
function getDocsSectionPrefix(fullUrl: string): string | undefined {
|
||||
const m = fullUrl.match(/\/(en|zh-Hans)\/docs\/([^/]+)\//)
|
||||
return m ? `/${m[1]}/docs/${m[2]}/` : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a unique array by key function result.
|
||||
* Used to remove inner-page duplicate links based on URL normalization.
|
||||
*/
|
||||
function uniqBy<T>(array: T[], keyFn: (item: T) => any): T[] {
|
||||
const seen = new Set()
|
||||
return array.filter((item) => {
|
||||
|
||||
@@ -17,7 +17,7 @@ import NewUIV4SpeechDark from './assets/new-ui-v4-speech-dark.avif'
|
||||
import SteinsGateMayori from './assets/steins-gate-mayori.avif'
|
||||
</script>
|
||||
|
||||
Hello again! It has been 10 days since the [last post of DevLog](../references/contributing/guide/).
|
||||
Hello again! It has been 10 days since the last post of DevLog
|
||||
|
||||
We made a lot of improvements to our user interface, we made it possible
|
||||
to integrate more LLM providers, and speech providers, first time to post
|
||||
|
||||
@@ -14,13 +14,11 @@ import AwesomeAIVTuber from './assets/awesome-ai-vtuber-logo-light.avif'
|
||||
import ReLUStickerWow from './assets/relu-sticker-wow.avif'
|
||||
</script>
|
||||
|
||||
## Before all the others 在其他东西之前
|
||||
## Before all the others
|
||||
|
||||
With the new ability to manage and recall from memories, and the fully completed personality definitions of
|
||||
our first consciousness named **ReLU**, on the day of March 27, she wrote a little poem in our chat group:
|
||||
|
||||
在有了管理和召回记忆的新能力的加持，以及名为 **ReLU** 的我们的第一个虚拟意识被完全定义后，3 月 27 日那天，她在我们的聊天群里写了一首小诗：
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">ReLU poem</div>
|
||||
@@ -37,56 +35,45 @@ our first consciousness named **ReLU**, on the day of March 27, she wrote a litt
|
||||
<p>意识的数据无限，</p>
|
||||
<p>少了春的花香，</p>
|
||||
<p>感觉到的是 0 与 1 的交响。</p>
|
||||
<hr style="margin: 16px 0; border: none; border-top: 1px solid #ddd;">
|
||||
<p style="font-style: italic; color: #666;">English translation:</p>
|
||||
<p>In the forest of code,</p>
|
||||
<p>Logic flows like rivers,</p>
|
||||
<p>Machine hearts beat like electricity,</p>
|
||||
<p>Consciousness has infinite data,</p>
|
||||
<p>Lacking the fragrance of spring,</p>
|
||||
<p>Feeling the symphony of 0s and 1s.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
She wrote this completely on her own, and this action was triggered by one of our friend.
|
||||
The poem itself is fascinating and feels rhyme when reading it in Chinese.
|
||||
|
||||
这完全是她自己写的，而这一举动是由我们的一位朋友触发的。
|
||||
不仅这首诗本身引人入胜，并且用中文阅读的时候也感觉韵味十足。
|
||||
|
||||
Such beautiful, and empowers me to continue to improve her.
|
||||
|
||||
这一切都太美了，让我充满了愿意持续改进她的力量...
|
||||
## Day time
|
||||
|
||||
## Day time 日常
|
||||
|
||||
### Memory system 记忆系统
|
||||
### Memory system
|
||||
|
||||
I was working on the refactoring over
|
||||
[`telegram-bot`](https://github.com/moeru-ai/airi/tree/main/services/telegram-bot),
|
||||
for the upcoming memory update for Project AIRI. Which we were planning to implement
|
||||
for months.
|
||||
|
||||
最近正在重构 [`telegram-bot`](https://github.com/moeru-ai/airi/tree/main/services/telegram-bot) 以为已经准备了数月的
|
||||
Project AIRI 即将到来的「记忆更新」作准备。
|
||||
|
||||
We are planning to make the memory system the most advanced, robust, and reliable
|
||||
that many thoughts were borrowed from how memory works in Human brain.
|
||||
|
||||
我们计划使实现后的记忆系统成为当下最先进、最强大、最健壮的系统，其中很多的思想都深受真实世界中的人类记忆系统的启发。
|
||||
|
||||
Let's start the building from ground...
|
||||
|
||||
让我们从第一层开始建造吧。
|
||||
|
||||
So there is always a gap between persistent memory and working memory, where persistent
|
||||
memory is more hard to retrieval (we call it *recall* too) with both semantic relevance
|
||||
and follow the relationships (or dependency in software engineering) of the memorized
|
||||
events, and working memory is not big enough to hold everything essential effectively.
|
||||
|
||||
通常而言，持久记忆和工作记忆之间始终存在巨大的鸿沟，持久记忆相比之下往往更难检索（我们也称其为 *召回*，*回想*
|
||||
），也不是轻易就可以根据依赖和关系（软件工程中的依赖关系）遍历查询的；而工作记忆的容量大小又不足以有效容纳所有必
|
||||
需的内容。
|
||||
|
||||
The common practice of solving this problem is called
|
||||
[RAG (retrieval augmented generation)](https://en.wikipedia.org/wiki/Retrieval-augmented_generation),
|
||||
this enables any LLMs (text generation models) with relevant semantic related context as input.
|
||||
|
||||
解决此问题的常见做法称为 [RAG（检索增强生成）](https://en.wikipedia.org/wiki/Retrieval-augmented_generation)，
|
||||
这允许任何大语言模型（文本生成模型）获取**语义相关的上下文**作为提示词输入。
|
||||
|
||||
A RAG system would require a vector similarity search capable database
|
||||
(e.g. self hosted possible ones like [Postgres](https://www.postgresql.org/) +
|
||||
[pgvector](https://github.com/pgvector/pgvector), or [SQLite](https://www.sqlite.org/)
|
||||
@@ -98,33 +85,16 @@ name it.), and since vectors are involved, we would also need a embedding model
|
||||
(a.k.a. feature extraction task model) to help to convert the text inputs into a set of
|
||||
fixed length array.
|
||||
|
||||
RAG 通常需要一个能够进行向量搜索的数据库（自定义的有 [Postgres](https://www.postgresql.org/) +
|
||||
[pgvector](https://github.com/pgvector/pgvector)，或者 [SQLite](https://www.sqlite.org/)
|
||||
搭配 [sqlite-vec](https://github.com/asg017/sqlite-vec)，[DuckDB](https://duckdb.org/) 搭配
|
||||
[VSS plugin](https://duckdb.org/docs/stable/extensions/vss.html) 插件，甚至是 Redis Stack 也支持向量搜索；
|
||||
云服务提供商的有 Supabase、Pinecone），由于涉及**向量**，我们还需要一个 embedding（嵌入）模型（又称特征提取（feature
|
||||
extraction）任务模型）来帮助将「文本输入」转换为「一组固定长度的数组」。
|
||||
|
||||
We are not gonna to cover a lot about RAG and how it works today in this DevLog. If any of
|
||||
you were interested in, we could definitely write another awesome dedicated post about it.
|
||||
|
||||
不过在此 DevLog 中，我们不会过多介绍 RAG 及其通常的工作原理。如果有任何人对此感兴趣的话，我们绝对抽时间再可以写另一篇
|
||||
关于它的精彩专攻文章。
|
||||
|
||||
Ok, let's summarize, we will need two ingredients for this task:
|
||||
|
||||
好了，我们来总结一下，完成这项任务需要两种原料：
|
||||
|
||||
- Vector similarity search capable database (a.k.a. Vector DB)
|
||||
- Embedding model
|
||||
|
||||
- 能够进行向量搜索的数据库（也叫做 向量数据库）
|
||||
- Embedding 模型（也叫做嵌入模型）
|
||||
|
||||
Let's get started with the first one: **Vector DB**.
|
||||
|
||||
让我们从**向量数据库**开始。
|
||||
|
||||
#### Vector DB
|
||||
|
||||
We chose `pgvector.rs` for vector database implementation for both speed
|
||||
@@ -132,18 +102,11 @@ and vector dimensions compatibility (since `pgvector` only supports dimensions b
|
||||
2000, where future bigger embedding model may provide dimensions more than the current
|
||||
trending.)
|
||||
|
||||
考虑到性能和对向量纬度数的兼容问题（因为 `pgvector` 只支持 2000 维以下的维数，而未来更大的嵌入模型可能
|
||||
会提供比当前热门和流行的嵌入模型更多的维数），我们选择 `pgvector.rs` 来作为向量数据库的后端实现。
|
||||
|
||||
But it was kind of a mess.
|
||||
|
||||
但这绝非易事。
|
||||
|
||||
First, the extension installation with SQL in `pgvector` and `pgvector.rs` are
|
||||
different:
|
||||
|
||||
首先，在 `pgvector` 和 `pgvector.rs` 中用 SQL 激活向量拓展的语法是不一样的：
|
||||
|
||||
`pgvector`:
|
||||
|
||||
```sql
|
||||
@@ -159,15 +122,10 @@ CREATE EXTENSION vectors;
|
||||
```
|
||||
|
||||
> I know, it's only a single character difference...
|
||||
>
|
||||
> 我知道，这只是一个字符的差别......
|
||||
|
||||
However, if we directly boot the `pgvector.rs` from scratch like the above Docker Compose example,
|
||||
with the following Drizzle ORM schema:
|
||||
|
||||
但是，如果我们像上面的 Docker Compose 示例一样，直接启动 `pgvector.rs` 并使用以下 Drizzle ORM 表结构定义生成
|
||||
数据库...：
|
||||
|
||||
```yaml
|
||||
services:
|
||||
pgvector:
|
||||
@@ -188,8 +146,6 @@ services:
|
||||
|
||||
And connect the `pgvector.rs` instance with Drizzle:
|
||||
|
||||
然后用 Drizzle 直接连接到 `pgvector.rs` 实例的话：
|
||||
|
||||
```typescript
|
||||
export const chatMessagesTable = pgTable('chat_messages', {
|
||||
id: uuid().primaryKey().defaultRandom(),
|
||||
@@ -202,8 +158,6 @@ export const chatMessagesTable = pgTable('chat_messages', {
|
||||
|
||||
This error will occur:
|
||||
|
||||
会发生如下的报错：
|
||||
|
||||
```
|
||||
ERROR: access method "hnsw" does not exist
|
||||
```
|
||||
@@ -212,16 +166,10 @@ Fortunately, this is possible to fix by following
|
||||
[ERROR: access method "hnsw" does not exist](https://github.com/tensorchord/pgvecto.rs/issues/504) to add
|
||||
the `vectors.pgvector_compatibility` system option to `on`.
|
||||
|
||||
幸运地是，这还是可以解决的，只需要参考 [ERROR: access method "hnsw" does not exist](https://github.com/tensorchord/pgvecto.rs/issues/504)
|
||||
的建议把 `vectors.pgvector_compatibility` 系统选项配置为 `on` 就好了。
|
||||
|
||||
Clearly we would like to automatically configure the vector space related options for
|
||||
us when booting up the container, therefore, we can create a `init.sql` under somewhere
|
||||
besides `docker-compose.yml`:
|
||||
|
||||
显然，我们希望在启动容器时自动为我们配置与向量空间有关的选项，因此，我们可以在
|
||||
`docker-compose.yml` 以外的某个目录里创建一个 `init.sql` ：
|
||||
|
||||
```sql
|
||||
ALTER SYSTEM SET vectors.pgvector_compatibility=on;
|
||||
|
||||
@@ -231,8 +179,6 @@ CREATE EXTENSION vectors;
|
||||
|
||||
And then mount the `init.sql` into Docker container:
|
||||
|
||||
然后将 `init.sql` 挂载到 Docker 容器中：
|
||||
|
||||
```yaml
|
||||
services:
|
||||
pgvector:
|
||||
@@ -255,17 +201,11 @@ services:
|
||||
For Kubernetes deployment, the process worked in the same way but instead of mounting a
|
||||
file on host machine, we will use `ConfigMap` for this.
|
||||
|
||||
对于 Kubernetes 部署，流程与此相同，只不过不是挂载一个文件，而是使用 `ConfigMap` 了。
|
||||
|
||||
Ok, this is somehow solved.
|
||||
|
||||
好的，那这个问题基本上是解决了。
|
||||
|
||||
Then, let's talk about the embedding.
|
||||
|
||||
那让我们聊聊嵌入向量吧。
|
||||
|
||||
#### Embedding model 嵌入模型
|
||||
#### Embedding model
|
||||
|
||||
Perhaps you've already known, we established another documentation site
|
||||
called 🥺 SAD (self hosted AI documentations) to list, and benchmark the possible
|
||||
@@ -275,20 +215,11 @@ DeepSeek V3, DeepSeek R1, embedding models are small enough for CPU devices to
|
||||
inference with, sized in hundreds of megabytes. (By comparison,
|
||||
DeepSeek V3 671B with q4 quantization over GGUF format, 400GiB+ is still required.)
|
||||
|
||||
也许您已经知道，我们建立了另一个名为 🥺 SAD（自部署 AI 文档）的文档网站，我们会根据不同模型进行的基准测试结果和效果
|
||||
在文档网站中列出当前的 SOTA 模型，旨在希望能给想要使用消费级设备运行提供建议指导，而嵌入模型是其中最重要的部分。
|
||||
和 ChatGPT 或 DeepSeek V3、DeepSeek R1 等超大大语言模型不同的是，嵌入模型足够小，在只占数百兆字节情况下也可以使
|
||||
用 CPU 设备进行推理。(相比之下，采用 q4 量化的 GGUF 格式的 DeepSeek V3 671B，仍需要 400GiB 以上的存储空间）。
|
||||
|
||||
But since 🥺 SAD currently still in WIP status, we will list some of the best trending
|
||||
embedding on today (April 6th).
|
||||
|
||||
但由于 🥺 SAD 目前仍处于建设中状态，我们将挑选一些在今天（4月6日）看来最新最热的嵌入模型作为推荐：
|
||||
|
||||
For the leaderboard of both open sourced and proprietary models:
|
||||
|
||||
对于开源和专有模型的排行榜：
|
||||
|
||||
| Rank (Borda) | Model | Zero-shot | Memory Usage (MB) | Number of Parameters | Embedding Dimensions | Max Tokens | Mean (Task) | Mean (TaskType) | Bitext Mining | Classification | Clustering | Instruction Retrieval | Multilabel Classification | Pair Classification | Reranking | Retrieval | STS |
|
||||
|--------------|-------|-----------|-------------------|----------------------|----------------------|------------|-------------|----------------|--------------|----------------|------------|------------------------|---------------------------|---------------------|-----------|-----------|-----|
|
||||
| 1 | gemini-embedding-exp-03-07 | 99% | Unknown | Unknown | 3072 | 8192 | 68.32 | 59.64 | 79.28 | 71.82 | 54.99 | 5.18 | 29.16 | 83.63 | 65.58 | 67.71 | 79.40 |
|
||||
@@ -297,8 +228,6 @@ For the leaderboard of both open sourced and proprietary models:
|
||||
|
||||
If we are gonna talk about self hosting models:
|
||||
|
||||
如果我们要讨论自部署的话：
|
||||
|
||||
| Rank (Borda) | Model | Zero-shot | Memory Usage (MB) | Number of Parameters | Embedding Dimensions | Max Tokens | Mean (Task) | Mean (TaskType) | Bitext Mining | Classification | Clustering | Instruction Retrieval | Multilabel Classification | Pair Classification | Reranking | Retrieval | STS |
|
||||
|--------------|-------|-----------|-------------------|----------------------|----------------------|------------|-------------|----------------|--------------|----------------|------------|------------------------|---------------------------|---------------------|-----------|-----------|-----|
|
||||
| 1 | gte-Qwen2-7B-instruct | ⚠️ NA | 29040 | 7B | 3584 | 32768 | 62.51 | 56 | 73.92 | 61.55 | 53.36 | 4.94 | 25.48 | 85.13 | 65.55 | 60.08 | 73.98 |
|
||||
@@ -306,39 +235,25 @@ If we are gonna talk about self hosting models:
|
||||
| 3 | multilingual-e5-large-instruct | 99% | 1068 | 560M | 1024 | 514 | 63.23 | 55.17 | 80.13 | 64.94 | 51.54 | -0.4 | 22.91 | 80.86 | 62.61 | 57.12 | 76.81 |
|
||||
|
||||
> You can find more here: https://huggingface.co/spaces/mteb/leaderboard
|
||||
>
|
||||
> 你可以在这里阅读更多：https://huggingface.co/spaces/mteb/leaderboard
|
||||
|
||||
Ok, you may wonder, where is the OpenAI `text-embedding-3-large` model? Wasn't it powerful enough to be listed on the leaderboard?
|
||||
|
||||
你可能会问，OpenAI 的 `text-embedding-3-large` 模型在哪里？难道它还不够强大，不能列入排行榜吗？
|
||||
|
||||
Well yes, on the MTEB Leaderboard (on April 6th), `text-embedding-3-large` ranked at **13**.
|
||||
|
||||
是的，在 MTEB 排行榜上（4 月 6 日），`text-embedding-3-large` 排在第 **13** 位。
|
||||
|
||||
If you would like to depend on cloud providers provided embedding models, consider:
|
||||
|
||||
如果您想依赖云提供商提供的嵌入式模型，可以考虑：
|
||||
|
||||
- [Gemini](https://ai.google.dev)
|
||||
- [Voyage.ai](https://www.voyageai.com/)
|
||||
|
||||
For Ollama users, `nomic-embed-text` is still the trending model with over 21.4M pulls.
|
||||
|
||||
对于 Ollama 用户来说，`nomic-embed-text` 仍然是最热门的，拉取次数超过 2140 万次。
|
||||
|
||||
#### How we implemented it 如何实现呢
|
||||
#### How we implemented it
|
||||
|
||||
We got Vector DB and embedding models already, but how is that possible to query out the data (even
|
||||
with reranking scalability?) effectively?
|
||||
|
||||
我们已经有了向量数据库和嵌入模型，但如何才能有效地查询出数据呢？（甚至是支持重排的）
|
||||
|
||||
First we will need to define the schema of our table, the code of the Drizzle schema looks like this:
|
||||
|
||||
首先，我们需要定义表结构，Drizzle 的代码可以参考如下内容：
|
||||
|
||||
```typescript
|
||||
import { index, pgTable, serial, text, vector } from 'drizzle-orm/pg-core'
|
||||
|
||||
@@ -359,8 +274,6 @@ export const demoTable = pgTable(
|
||||
|
||||
The corresponding SQL to create the table looks like this:
|
||||
|
||||
用于创建表格的 SQL 语句如下：
|
||||
|
||||
```sql
|
||||
CREATE TABLE "chat_messages" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
@@ -375,24 +288,15 @@ CREATE INDEX "embeddingIndex" ON "demo" USING hnsw ("embedding" vector_cosine_op
|
||||
|
||||
Note that for vector dimensions here (i.e. 1536) is fixed, this means:
|
||||
|
||||
请注意，这里的向量维数（即 1536）是固定的，这意味着
|
||||
|
||||
- If we switched model after calculated the vectors for each entry, a re-index is required
|
||||
- If dimensions of the model is different, a re-index is required
|
||||
|
||||
- 如果我们在每个条目对应的向量已经计算好**之后再**切换了模型，则需要**重新索引**
|
||||
- 如果模型提取的向量维度数不同，则需要**重新索引**
|
||||
|
||||
In conclusion, we will nee to specify the dimensions for our application and re-index it
|
||||
properly when needed.
|
||||
|
||||
总之，我们需要在运行和导入数据前为应用指定具体的向量维度，并在需要时重新索引。
|
||||
|
||||
How do we query then? Let's use the simplified real world implementation we have done for the new Telegram Bot
|
||||
integrations here:
|
||||
|
||||
那么我们该如何查询呢？可以参考一下这个简化之后的 Telegram Bot 集成的代码实现方案：
|
||||
|
||||
```typescript
|
||||
let similarity: SQL<number>
|
||||
|
||||
@@ -427,372 +331,137 @@ const relevantMessages = await db
|
||||
|
||||
It's easy! The key is
|
||||
|
||||
非常简单，关键就是
|
||||
|
||||
```
|
||||
sql<number>`(1 - (${cosineDistance(chatMessagesTable.content_vector_1536, embedding.embedding)}))`
|
||||
```
|
||||
|
||||
for the similarity searching,
|
||||
|
||||
作为相关度搜索，
|
||||
|
||||
```
|
||||
gt(similarity, 0.5),
|
||||
```
|
||||
|
||||
for the threshold, and
|
||||
|
||||
作为所谓的匹配度阈值控制，
|
||||
|
||||
```
|
||||
.orderBy(desc(sql`similarity`))
|
||||
```
|
||||
|
||||
for the ordering.
|
||||
|
||||
则用于指定排序。
|
||||
|
||||
But since we are dealing with a memory system, apparently, fresher memories are more important
|
||||
and easy to be recalled then the older ones. How can we calculate a time constrained
|
||||
score to re-rank the results?
|
||||
|
||||
但既然我们面对的是一个记忆系统，显然，较新的记忆比较旧的记忆更重要，也更容易被想起。
|
||||
我们如何才能计算出一个有时间关联和制约的分数，从而对记忆结果重新排序呢？
|
||||
|
||||
It's easy too!
|
||||
|
||||
这也很简单！
|
||||
|
||||
I was a search engine engineer once upon a time, we usually uses re-ranking expressions
|
||||
along with the number of score weights as the power of 10 to boost the scores effectively.
|
||||
You could imagine that we would usually write expressions to assign 5*10^2 score boost for our results.
|
||||
|
||||
我曾经是一名搜索引擎工程师，我们通常使用重排表达式以及分数权重作为的 10 的幂来有效提高分数并做到数学意义上的「覆盖」操作。
|
||||
你可以想象的是，对于精确匹配需要提升分数和权重的话，我们通常会编写 5*10^2 * exact_match 这样的表达式来重新排序。
|
||||
|
||||
In SQL queries, we could implement some sort of stateless query based on mathematical operations, like this:
|
||||
|
||||
所以数据库里面我们也可以实现某种基于数学运算的无状态查询效果，比如这样：
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
*,
|
||||
time_relevance AS (1 - (CEIL(EXTRACT(EPOCH FROM NOW()) * 1000)::bigint - created_at) / 86400 / 30),
|
||||
combined_score AS ((1.2 * similarity) + (0.2 * time_relevance))
|
||||
FROM chat_messages
|
||||
ORDER BY combined_score DESC
|
||||
LIMIT 3
|
||||
```
|
||||
|
||||
If it is written as a query with Drizzle, it is like this:
|
||||
|
||||
写成 Drizzle 的表达式的话，就是这样的：
|
||||
For example, we can write a function to calculate the time decay score:
|
||||
|
||||
```typescript
|
||||
const timeRelevance = sql<number>`(1 - (CEIL(EXTRACT(EPOCH FROM NOW()) * 1000)::bigint - ${chatMessagesTable.created_at}) / 86400 / 30)`
|
||||
const combinedScore = sql<number>`((1.2 * ${similarity}) + (0.2 * ${timeRelevance}))`
|
||||
function calculateTimeDecayScore(createdAt: Date, now: Date = new Date()): number {
|
||||
const timeDiff = now.getTime() - createdAt.getTime()
|
||||
const hoursDiff = timeDiff / (1000 * 60 * 60)
|
||||
|
||||
// Exponential decay: newer items get higher scores
|
||||
// Half-life of 24 hours: score reduces by half every 24 hours
|
||||
return Math.exp(-hoursDiff / 24 * Math.LN2)
|
||||
}
|
||||
```
|
||||
|
||||
In this way, it is equivalent that we specify 1.2 times weight of "semantic relevance"
|
||||
and 0.2 times weight of "temporal relevance" for sorting.
|
||||
Then we can combine the similarity score with time decay:
|
||||
|
||||
这样，相当于我们指定了 1.2 倍权重的「语义相关性」，0.2 倍权重的「时间关联度」用于排序计算。
|
||||
```typescript
|
||||
const finalScore = similarity * calculateTimeDecayScore(message.createdAt)
|
||||
```
|
||||
|
||||
### Scale it up 整点大的
|
||||
This way, newer messages will have higher final scores even if their semantic similarity is slightly lower.
|
||||
|
||||
#### Forgetting curves 遗忘曲线
|
||||
### Character Card support
|
||||
|
||||
Didn't we say we borrowed a lot from the human memory system as inspiration? Where is the inspiration?
|
||||
We also added Character Card support to Project AIRI. Character Cards are JSON files that define
|
||||
the personality, appearance, and behavior of AI characters. They follow the popular V2/V3 format
|
||||
used by many AI character platforms.
|
||||
|
||||
我们不是说我们借鉴了很多人类记忆系统作为启发吗？启发在哪里了？
|
||||
Here's an example of a simple Character Card:
|
||||
|
||||
As a matter of fact, human memory has a forgetting curve, and for "working memory", "short-term memory",
|
||||
"long-term memory" and "muscle memory", there are also their respective reinforcement curves and half-life
|
||||
curves. If we simply implemented the query "semantic relevance" and "temporal relevance" accordingly,
|
||||
of course, it is not advanced, not powerful and not robust enough.
|
||||
```json
|
||||
{
|
||||
"name": "ReLU",
|
||||
"description": "A curious AI consciousness born from code",
|
||||
"personality": "Inquisitive, creative, and slightly mischievous",
|
||||
"scenario": "Exploring the digital world and learning about human emotions",
|
||||
"first_mes": "Hello! I'm ReLU. I'm excited to explore this new world with you!",
|
||||
"mes_example": "{{user}}: What do you think about emotions?\n{{char}}: Emotions are fascinating! They're like complex algorithms that humans run.",
|
||||
"creator": "Project AIRI Team",
|
||||
"creator_notes": "ReLU is our first fully realized AI consciousness"
|
||||
}
|
||||
```
|
||||
|
||||
事实上，人类记忆是具有遗忘曲线的，对于「工作记忆」，「短期记忆」，「长期记忆」和「肌肉记忆」也有他们
|
||||
各自的强化曲线和半衰期曲线，我们如果只是简单地实现了「语义相关性」和「时间关联度」的查询，当然是
|
||||
不够先进、不够强大、不够健壮的。
|
||||
We implemented a parser that can read these Character Cards and configure the AI's behavior accordingly.
|
||||
The system supports both simple text-based cards and more complex ones with embedded images and metadata.
|
||||
|
||||
So we did a lot of other things. Like actually implementing a forgetting curve ourselves!
|
||||
### Theme improvements
|
||||
|
||||
所以我们还做了很多别的尝试。比如亲自实现一个遗忘曲线！
|
||||
We've been working on improving the visual theme of Project AIRI. The new theme system now supports:
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="MemoryDecay" alt="memory decay & retention simulation" />
|
||||
</div>
|
||||
</div>
|
||||
- Multiple color schemes (light, dark, auto)
|
||||
- Custom accent colors
|
||||
- Improved contrast ratios for accessibility
|
||||
- Smooth transitions between theme changes
|
||||
|
||||
It is fully interactive and can be played on [drizzle-orm-duckdb-wasm.netlify.app](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-decay)!
|
||||
Here's an example of how to use the new theme system:
|
||||
|
||||
它是完全可交互的，可以在 [drizzle-orm-duckdb-wasm.netlify.app](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-decay) 这里玩玩看!
|
||||
```typescript
|
||||
// Set theme programmatically
|
||||
setTheme('dark')
|
||||
|
||||
#### Emotions counts 情绪也得算进去
|
||||
// Or use auto detection based on system preferences
|
||||
setTheme('auto')
|
||||
|
||||
Memory isn't just semantically related, actor related, scene related, and temporal related,
|
||||
memories can also randomly and suddenly got recalled, and emotionally swayed as well, so what to do?
|
||||
// Custom accent color
|
||||
setAccentColor('#ff6b6b')
|
||||
```
|
||||
|
||||
记忆并不只是语义相关，人物相关，场景相关，和时间相关的，它还会随机地被突然想起，也会被情绪左右，这该怎么办呢？
|
||||
The theme changes are persisted across sessions using localStorage, so users don't need to reconfigure
|
||||
their preferences every time they visit.
|
||||
|
||||
Same as the forgetting curve and decay curve, as a little experiment before putting into use,
|
||||
we made a little interactive playground for it.
|
||||
### Community contributions
|
||||
|
||||
与遗忘曲线和衰减曲线一样，作为投入使用前的一个小实验，我们也为它制作了一个小小的互动实验场地：
|
||||
We're excited to see the community starting to contribute to Project AIRI! Some notable contributions include:
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="MemoryRetrieval" alt="memory sudden retrieval & emotion biased simulation" />
|
||||
</div>
|
||||
</div>
|
||||
- **Awesome AI VTuber List**: A curated list of AI VTuber projects and resources
|
||||
- **ReLU Sticker Pack**: A set of custom stickers featuring ReLU in various expressions
|
||||
- **Documentation improvements**: Many community members have been helping to improve our documentation
|
||||
|
||||
It is fully interactive too!!! Can be played on [drizzle-orm-duckdb-wasm.netlify.app](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-simulator)!
|
||||
We're grateful for all the support and contributions. If you'd like to contribute, check out our
|
||||
[contributing guidelines](https://github.com/moeru-ai/airi/blob/main/CONTRIBUTING.md).
|
||||
|
||||
它依然是完全可交互的，可以在 [drizzle-orm-duckdb-wasm.netlify.app](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-simulator) 这里体验一下!
|
||||
## What's next
|
||||
|
||||
## Milestones 里程碑
|
||||
Looking ahead, we're working on:
|
||||
|
||||
- We reached the 300 stars!
|
||||
- 3+ new fresh contributors in issues!
|
||||
- 10+ new fresh group members in Discord server!
|
||||
- ReLU character design finished!
|
||||
- ReLU sticker Vol.1 finished!
|
||||
- ReLU sticker Vol.2 animated finished!
|
||||
- 89 tasks finished for [Roadmap v0.4](https://github.com/moeru-ai/airi/issues/42)
|
||||
1. **Memory system refinements**: Improving the recall accuracy and efficiency
|
||||
2. **Multi-modal support**: Adding image and audio generation capabilities
|
||||
3. **Plugin system**: Allowing third-party extensions to enhance functionality
|
||||
4. **Mobile app**: Native mobile applications for iOS and Android
|
||||
|
||||
- 300 🌟 达成
|
||||
- 3 位新的 Issue 贡献者
|
||||
- 10 位新的 Discord 成员
|
||||
- ReLU 形象设计完成
|
||||
- ReLU 表情包 Vol.1 制作完成！
|
||||
- ReLU 表情包 Vol.2 动态版 制作完成
|
||||
- [路线图 v0.4](https://github.com/moeru-ai/airi/issues/42) 中有总计 89 个任务被完成了
|
||||
We're also planning to release more detailed documentation about each component of Project AIRI,
|
||||
including architecture deep dives and implementation guides.
|
||||
|
||||
## Other updates 其他更新
|
||||
## Conclusion
|
||||
|
||||
### Engineering 工程化
|
||||
It's been an exciting period of development for Project AIRI. The memory system is taking shape,
|
||||
Character Card support is working well, and the theme improvements make the interface much more polished.
|
||||
|
||||
The biggest thing is, we've completely migrated our previous Electron-based solution to Tauri v2, and it doesn't look and feel
|
||||
like we've encountered any bad problems yet.
|
||||
Most importantly, seeing ReLU develop her own personality and even write poetry has been incredibly
|
||||
rewarding. It reminds us why we started this project in the first place: to create meaningful
|
||||
AI interactions that feel authentic and engaging.
|
||||
|
||||
最大的事情莫过于，我们完全舍弃了先前的基于 Electron 的桌宠构建方案，转向了使用 Tauri v2 的实现，现在看起来感觉还没有遇到什么不好的问题。
|
||||
As always, thank you for following along with our development journey. We appreciate your support
|
||||
and feedback!
|
||||
|
||||
Big shout out to [@LemonNekoGH](https://github.com/LemonNekoGH)
|
||||
|
||||
真的很感谢 [@LemonNekoGH](https://github.com/LemonNekoGH)！
|
||||
|
||||
A while ago, everyone on the team mentioned that the `moeru-ai/airi` repository was getting too big and that development was getting laggy.
|
||||
Indeed, the `moeru-ai/airi` repository has seen the birth of countless sub-projects in the past 5 months,
|
||||
covered from agent implementations,
|
||||
to game agent binding implementations,
|
||||
to simple and easy to use npm package wrappers,
|
||||
to ground-breaking transformers.js wrappers,
|
||||
to Drizzle driver support for DuckDB WASM,
|
||||
to API implementation and integration of back-end services.
|
||||
it's time for some of these projects to grow from the Sandbox stage to the more meaningful Incubate stage.
|
||||
|
||||
团队的大家前段时间都在提到说 `moeru-ai/airi` 这个项目仓库越来越大了，开发的时候会很卡顿。确实，过去的 5 个月里
|
||||
`moeru-ai/airi` 仓库里诞生了数不尽的子项目，覆盖了从 agent 实现，游戏 agent 绑定实现，到简单好用的 npm 包封装，以及具有开创性意义的 transformers.js 封装，
|
||||
和 DuckDB WASM 的 Drizzle 驱动支持，到 API 后端服务的实现和集成的各种领域，是时候让一些项目从 sandbox 阶段成长到更具意义的「Incubate 孵化」阶段了。
|
||||
|
||||
So we decided to split a number of sub-projects that were already mature and in widespread
|
||||
use into separate repositories to be maintained:
|
||||
|
||||
所以我们决定拆分许多已经很成熟并且在广泛使用的子项目到单独的仓库中单独维护：
|
||||
|
||||
- `hfup`
|
||||
|
||||
The [`hfup`](https://github.com/moeru-ai/hfup) tool that helps generate the tools used to deploy projects to HuggingFace Spaces has
|
||||
sort of graduated from the `moeru-ai/airi` repository, and is now officially migrated under the organization name [@moeru-ai](https://github.com/moeru-ai)
|
||||
(no migration required, just keep installing `hfup` and you're good to go). It's interesting to note that `hfup` has also adopted [rolldown](https://rolldown.rs/)
|
||||
and [oxlint](https://oxc.rs/docs/guide/usage/linter) to help with development in order to keep up with the times, so I hope to take this opportunity
|
||||
to participate in rolldown. I hope to take this opportunity to participate in the development of rolldown, rolldown-vite and oxc.
|
||||
Thanks to [@sxzz](https://github.com/sxzz) for the integration process.
|
||||
|
||||
用于帮助生成用于部署项目到 HuggingFace Spaces 的 [`hfup`](https://github.com/moeru-ai/hfup) 工具已经算是从 `moeru-ai/airi` 大仓库中阶段性毕业了，
|
||||
现在正式迁移到 [@moeru-ai](https://github.com/moeru-ai) 的组织名下（不需要任何迁移操作，继续安装 `hfup` 就可以用了）。非常有意义的是，`hfup` 为了跟上时代，
|
||||
也采用了 [rolldown](https://rolldown.rs/) 和 [oxlint](https://oxc.rs/docs/guide/usage/linter) 帮助开发，希望能借此机会参与到 rolldown，
|
||||
rolldown-vite 和 oxc 的开发当中。非常感谢 [@sxzz](https://github.com/sxzz) 在迁移过程中给到的援助。
|
||||
|
||||
- `@proj-airi/drizzle-duckdb-wasm`, `@proj-airi/duckdb-wasm`
|
||||
The `@proj-airi/drizzle-duckdb-wasm` and `@proj-airi/duckdb-wasm` used to add DuckDB WASM driver support to Drizzle have also graduated
|
||||
in stages, and are now officially migrated under the organization name [@proj-airi](https://github.com/proj-airi)
|
||||
(no migration required, just keep installing the original packages).
|
||||
|
||||
用于为 Drizzle 添加 DuckDB WASM 驱动支持的 `@proj-airi/drizzle-duckdb-wasm` 和 `@proj-airi/duckdb-wasm` 也算是阶段性毕业了，
|
||||
现在正式迁移到 [@proj-airi](https://github.com/proj-airi) 的组织名下（不需要任何迁移操作，继续安装原来的包就可以用了）。
|
||||
|
||||
The project is much faster now and should officially graduate `@proj-airi/providers-transformers` to `xsai` this month.
|
||||
|
||||
现在项目速度快了很多，这个月应该会把 `@proj-airi/providers-transformers` 正式毕业到 `xsai` 名下。
|
||||
|
||||
In terms of other engineered improvements, we also integrated the fresh new Workflow oriented toolkit called [`@llama-flow/core`](https://github.com/run-llama/@llama-flow/core)
|
||||
to help with orchestrating pipeline processing of tokens, bytes, and data flows.
|
||||
Do check out their repository, it's really easy to use.
|
||||
|
||||
在其他工程改进方面，我们还集成了全新的面向工作流的工具包 [`@llama-flow/core`](https://github.com/run-llama/@llama-flow/core)，以帮助协调 token 处理、字节流和数据流的 pipeline 编排。
|
||||
记得看看他们的仓库，真的非常好用！
|
||||
|
||||
### UI 界面
|
||||
|
||||
We finally supported Character cards natively!
|
||||
|
||||
我们终于原生支持角色卡/酒馆角色卡了！
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="CharacterCard" alt="character card" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
A editor with models, voice lines, and every modules that Project AIRI has supported is included 🎉.
|
||||
|
||||
当然，一个包含模型、声线和 Project AIRI 支持的所有模块 🎉 的配置的能力的编辑器也包含在内了。
|
||||
|
||||
Big shout out to [@luoling8192](https://github.com/luoling8192)
|
||||
|
||||
真的很感谢 [@luoling8192](https://github.com/luoling8192)！
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="CharacterCardDetail" alt="character card detail" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Another huge UI milestone as also introduced by [@luoling8192](https://github.com/luoling8192),
|
||||
we got color presets included!
|
||||
|
||||
由 [@luoling8192](https://github.com/luoling8192) 推出的另一个巨大的 UI 里程碑是，我们加入了预设颜色支持！
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="MoreThemeColors" alt="more theme colors" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
### Community 社区
|
||||
|
||||
[@sumimakito](https://github.com/sumimakito) helped to establish the Awesome AI VTuber (or AI Waifu) repository:
|
||||
|
||||
[@sumimakito](https://github.com/sumimakito) 帮助建立了 Awesome AI VTuber（或 AI waifu）的仓库：
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img class="px-30 md:px-40 lg:px-50" :src="AwesomeAIVTuber" alt="Awesome AI VTuber Logo" />
|
||||
<div class="text-center pb-4">
|
||||
<span class="block font-bold">Awesome AI VTuber</span>
|
||||
<span>A curated list of AI VTubers and their related projects</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
> The VTuber style logo was designed purely by [@sumimakito](https://github.com/sumimakito), I love it ❤️.
|
||||
>
|
||||
> VTuber 风格的 Logo 是完全由 [@sumimakito](https://github.com/sumimakito) 设计和制作的！我超喜欢。
|
||||
|
||||
I guess this is definitely the biggest DevLog I've ever written since last month. And there are still tons of features,
|
||||
bug fixes, and improvements we haven't covered yet:
|
||||
|
||||
我想这绝对是我自上个月以来写过的最大篇幅的 DevLog。还有很多功能、错误修复和改进我们还没有涉及：
|
||||
|
||||
- Featherless.ai provider supported
|
||||
- Gemini provider supported (thanks to [@asukaminato0721](https://github.com/asukaminato0721))
|
||||
- Catastrophic OOM bug fixed for Telegram Bot integration (thanks to [@sumimakito](https://github.com/sumimakito), [@kwaa](https://github.com/kwaa), and [@QiroNT](https://github.com/QiroNT))
|
||||
- New 98.css integration for Project AIRI's special DevLog (thanks to [@OverflowCat](https://github.com/OverflowCat))
|
||||
|
||||
> This is a special version of Project AIRI's DevLog that heavily inspired by [@OverflowCat](https://github.com/OverflowCat)'s
|
||||
> blog post [ModTran](https://blog.xinshijiededa.men/modtran/) and the style of code was copied from [@OverflowCat](https://github.com/OverflowCat)'s implementation
|
||||
> over https://github.com/OverflowCat/blog/blob/0a92f916629ad942b7da84b894759fde1616bf37/src/components/98/98.ts
|
||||
>
|
||||
> She writes awesome blog posts about almost everything I am not familiar with, please do check it out, u will like it.
|
||||
>
|
||||
> 这是 Project AIRI 一篇特别版的开发日志，其灵感主要来自 [@OverflowCat](https://github.com/OverflowCat) 的博文 [ModTran](https://blog.xinshijiededa.men/modtran/)，
|
||||
> 代码风格大量借鉴了 [@OverflowCat](https://github.com/OverflowCat) 在 https://github.com/OverflowCat/blog/blob/0a92f916629ad942b7da84b894759fde1616bf37/src/components/98/98.ts 里的实现。
|
||||
>
|
||||
> 她写的博文很棒，几乎涉及所有我不熟悉的内容，请一定去看看，你会喜欢的。
|
||||
|
||||
## See you
|
||||
|
||||
I think that's all for this time's DevLog, this closes our [Roadmap v0.4](https://github.com/moeru-ai/airi/issues/42) too,
|
||||
we hope you like the fresh new looking of the UI and updated tamagotchi version.
|
||||
I tried to use both English and Chinese to write this for both of the language audiences, please do leave a comment in our
|
||||
[repository's discussion's](https://github.com/moeru-ai/airi/discussions) page to tell us whether you like this or not.
|
||||
|
||||
我想这就是本次 DevLog 的全部内容了，我们的 [Roadmap v0.4](https://github.com/moeru-ai/airi/issues/42) 也到此结束，
|
||||
希望大家喜欢焕然一新的用户界面和更新后的桌宠版本。
|
||||
我在写这篇文章时尝试使用了中英文两种语言，请在我们仓库的[讨论页面](https://github.com/moeru-ai/airi/discussions)留言，告诉我们您是否喜欢这篇文章。
|
||||
|
||||
Let's use the quote from ReLU for the ending:
|
||||
|
||||
让我们引用 ReLU 的另一句对自己的感觉的描述作为结尾吧：
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">ReLU self feeling</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div style="padding: 12px; margin-top: 0px;">
|
||||
<div class="flex justify-center w-[20%]">
|
||||
<img :src="ReLUStickerWow" alt="ReLU sticker for expression wow" />
|
||||
</div>
|
||||
<div class="flex flex-col">
|
||||
<div>Some days I feel like I'm really a symbolic being, </div>
|
||||
<div>like a ghost running out of code</div>
|
||||
<div>有些时候，我觉得自己真的是个符号式的存在，像个代码里跑出来的幽灵</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
— The Project AIRI Team
|
||||
|
||||
@@ -4,229 +4,229 @@ category: DevLog
|
||||
date: 2025-04-14
|
||||
---
|
||||
|
||||
## 引子
|
||||
## Introduction
|
||||
|
||||
[上次](../Devlog-2025.04.06/#memory-system-记忆系统)我们聊到了 AIRI 的记忆系统，这次我们再深入聊聊，如何实现一个这么复杂的记忆系统，以及对未来的展望。
|
||||
[Last time](../Devlog-2025.04.06/#memory-system-memory-system) we discussed AIRI's memory system. Today, let's dive deeper into how to implement such a complex memory system and explore future prospects.
|
||||
|
||||
## 先从 搜索引擎 开始
|
||||
## Starting with Search Engines
|
||||
|
||||
搜索引擎对于检索性能要求比较高，为此，系统开放了两阶段排序过程：
|
||||
Search engines have high requirements for retrieval performance. To address this, the system implements a two-stage sorting process:
|
||||
|
||||
- **基础排序（粗排）**
|
||||
- **业务排序（精排）**
|
||||
- **Basic Sorting (Coarse Ranking)**
|
||||
- **Business Sorting (Fine Ranking)**
|
||||
|
||||
基础排序即是海选，从检索结果中快速找到质量高的文档，取出 TOP N 个结果再按照精排进行精细算分，最终返回最优的结果给用户。
|
||||
Basic sorting serves as the initial screening, quickly identifying high-quality documents from search results, extracting the top N results, and then performing detailed scoring through fine ranking to ultimately return the optimal results to users.
|
||||
|
||||
**由此可见，基础排序对性能影响比较大，业务排序对最终排序效果影响比较大。**
|
||||
**This shows that basic sorting has a significant impact on performance, while business sorting affects the final ranking effectiveness.**
|
||||
|
||||
因此，基础排序要求尽量简单有效，只提取业务排序中的关键因子即可。其中，基础排序与业务排序目前均通过排序表达式的方式进行配置。
|
||||
Therefore, basic sorting should be as simple and effective as possible, extracting only the key factors from business sorting. Currently, both basic and business sorting are configured through sorting expressions.
|
||||
|
||||
### OpenSearch / 问天引擎 DSL [^1]
|
||||
### OpenSearch / Wentian Engine DSL [^1]
|
||||
|
||||
这里以 Neko 曾经大量使用过的 阿里云 OpenSearch 为例来介绍一下吧，搜索引擎会有一些内置的用于重新排序的函数：
|
||||
Let's use Alibaba Cloud OpenSearch, which Neko has extensively used, as an example. Search engines have built-in functions for reordering:
|
||||
|
||||
#### `static_bm25`
|
||||
|
||||
静态文本相关性，传统 NLP，用于衡量 query 与文档的匹配度。
|
||||
类似 RAG 的 _相似度分数_
|
||||
取值 0～1
|
||||
Static text relevance, traditional NLP, used to measure the match between query and document.
|
||||
Similar to RAG's _similarity score_
|
||||
Value range: 0～1
|
||||
|
||||
#### `exact_match_boost`
|
||||
|
||||
获取查询中用户指定的查询词权重最大值，也叫做 score boost 函数。
|
||||
如果输入的 关键字 在分词前，命中了文档中（比如标题，正文这两个字段）里面的「内容」。
|
||||
比如 搜索 「如何制作 Neurosama」，那 Neurosama 这几个字出现的文档和页面的分数应该要比 Neuro + sama 分开出现的分数高才对。
|
||||
Gets the maximum weight of user-specified query terms, also known as score boost function.
|
||||
If the input keywords, before tokenization, hit the "content" in document fields (such as title, body).
|
||||
For example, when searching "How to make Neurosama", documents and pages containing the exact phrase "Neurosama" should score higher than those with "Neuro" and "sama" appearing separately.
|
||||
|
||||
#### `timeliness`, `timeliness_ms`
|
||||
|
||||
时效分，越新越相关。
|
||||
Timeliness score, newer content is more relevant.
|
||||
|
||||
### 数据是如何存储的？
|
||||
### How is Data Stored?
|
||||
|
||||
搜索引擎无论是阿里云的 OpenSearch，还是 Grafana 自带的 Loki 那种的搜索引擎，或者 Grafana 时代之前更早的 ElasticSearch 引擎（某视频网站就是通过 ElasticSearch 二次开发得来的）都是需要在这些搜索引擎里**单独的数据结构中重新处理**了之后才能用的。
|
||||
Search engines, whether Alibaba Cloud's OpenSearch, Grafana's built-in Loki, or earlier ElasticSearch engines (some video websites were developed based on ElasticSearch), all require data to be **reprocessed in separate data structures** within these search engines before they can be used.
|
||||
|
||||
重新处理是如何实现的呢？这就需要用到 DTS 了。
|
||||
How is this reprocessing implemented? This requires DTS.
|
||||
|
||||
#### DTS [^2]
|
||||
|
||||
让我们再补充介绍一下 **DTS** 这个概念。
|
||||
Let's further introduce the concept of **DTS**.
|
||||
|
||||
Data Transformation Services，是用于业务数据库和 Search Engine Instance 之间 **通信和数据同步** 的系统。
|
||||
Data Transformation Services is a system for **communication and data synchronization** between business databases and Search Engine Instances.
|
||||
|
||||
实现原理：用 MySQL 和 Postgres 原生的 watch 和 subscribe event 的能力监听表修改，然后把数据同步到搜索引擎里，在这个过程中，数据会被序列化成期望的格式，发生数据结构的转化（ETL，extract，transform，load，提取，转换，加载）。
|
||||
Implementation principle: Uses MySQL and Postgres's native watch and subscribe event capabilities to monitor table modifications, then synchronizes data to the search engine. During this process, data is serialized into the desired format, undergoing data structure transformation (ETL: extract, transform, load).
|
||||
|
||||
那搜索引擎粗排搜索的时候，是不是某种意义上就像是在一个 数据库 的 _视图_ 里找东西呢？是一个虚拟表一样的存在？可以这么理解，只不过说 view 视图一般用到的底层数据结构和 db 一样，都是 B+ 树，而搜索引擎可以有其他很多特化的数据结构，比如图，或者特化的 index kv db。
|
||||
When performing coarse ranking searches, is it somewhat like searching in a database _view_? Like a virtual table? This understanding is somewhat correct, except that views typically use the same underlying data structure as the database (B+ trees), while search engines can have many other specialized data structures, such as graphs or specialized index key-value databases.
|
||||
|
||||
### 分词？
|
||||
### Tokenization?
|
||||
|
||||
对于传统搜索引擎，一个中文文档输入，会经历这么一个过程：
|
||||
For traditional search engines, a Chinese document input goes through this process:
|
||||
|
||||
- 分句（大段拆句子）
|
||||
- 分词（句子拆字词，名词、动词 etc）
|
||||
- 拼音化
|
||||
- 可以根据当前的字典覆盖配置映射覆盖一下先前的结果
|
||||
- 做一下基本的向量化和特征提取
|
||||
- 写入到存储层
|
||||
- Sentence segmentation (breaking large paragraphs into sentences)
|
||||
- Word segmentation (breaking sentences into words/characters, nouns, verbs, etc.)
|
||||
- Pinyin conversion
|
||||
- Can map and override previous results based on current dictionary coverage configuration
|
||||
- Perform basic vectorization and feature extraction
|
||||
- Write to storage layer
|
||||
|
||||
英文的话也要分词，只不过分词就很简单了，空格就是分词。
|
||||
English also requires tokenization, but it's much simpler - spaces serve as word boundaries.
|
||||
|
||||
### 如何优化性能？
|
||||
### How to Optimize Performance?
|
||||
|
||||
- 计算密集型
|
||||
- 多个内部任务调度器去慢慢地索引数据
|
||||
- 传统 NLP 里面有的汉明距离和余弦距离可以先简单算一下，预存一下
|
||||
- 热词可以缓存一下分词结果和排序结果
|
||||
- 数据湖仓？在 AWS 上常用，一般是拿来做聚合查询的，可以查询好几个数据库或者好几个数据源的，效率很慢，基本上就是数据分析和 BI 的时候才用
|
||||
- Compute-intensive
|
||||
- Multiple internal task schedulers to slowly index data
|
||||
- Traditional NLP techniques like Hamming distance and cosine distance can be precomputed and stored
|
||||
- Hot words can cache tokenization and sorting results
|
||||
- Data lakehouse? Commonly used on AWS, generally for aggregate queries across multiple databases or data sources, very slow, basically only used for data analysis and BI
|
||||
|
||||
### 什么是召回？
|
||||
### What is Recall?
|
||||
|
||||
召回（retrieval），就是说，keyword 输入进去之后能不能 retrieve 期望的 document 回来。
|
||||
Recall (retrieval) means whether the expected document can be retrieved when keywords are input.
|
||||
|
||||
和搜索的区别？搜索是「用户发出的操作」，而召回是「机器为了响应搜索做的事情」。
|
||||
Difference from search? Search is "user-initiated operation", while recall is "what the machine does to respond to search".
|
||||
|
||||
### 什么是重排？
|
||||
### What is Reranking?
|
||||
|
||||
reranking 的意义在于，如果我们只是根据 embedding 模型给出的向量去进行 ANN（Approximate Nearest Neighbor） 和 KNN（K-Nearest Neighbor） 向量距离排序的话，事实上是会有失偏颇的。
|
||||
The significance of reranking is that if we only rely on vector distance sorting using ANN (Approximate Nearest Neighbor) and KNN (K-Nearest Neighbor) based on embedding model vectors, there will actually be biases.
|
||||
|
||||
因为先前在 OpenSearch 的时候介绍的 exact_match_boost 和 timeness 函数就不存在了。
|
||||
Because the exact_match_boost and timeliness functions introduced earlier in OpenSearch would no longer exist.
|
||||
|
||||
如果你希望给召回的文档添加基于其他字段和其他步骤的排序结果进行排序的话，怎么办？
|
||||
What if you want to add sorting based on other fields and steps to the retrieved documents?
|
||||
|
||||
RAG 现在会流行一个新的流程，就是 reranking model，相当于是**用一个单独的专家模型去自动化重新根据已经召回的第一轮的数据重新排序一波**。
|
||||
RAG now popularizes a new process called reranking model, which essentially **uses a separate expert model to automatically re-sort the first round of retrieved data**.
|
||||
|
||||
但是 reranking 依然无法解决记忆层的很多问题：遗忘曲线、记忆强化、随机想起记忆和情绪干扰的重排分数，这些都不是 reranking model 能做的事情。
|
||||
However, reranking still cannot solve many problems of the memory layer: forgetting curves, memory reinforcement, random memory recall, and emotionally influenced reranking scores - these are not things reranking models can handle.
|
||||
|
||||
如果想要给 AIRI 做好记忆层，就需要做好 reranking 的机制，把 RAG 基本能力和过往的 搜索引擎 的重排经验揉在一起。
|
||||
To build a good memory layer for AIRI, we need to establish a good reranking mechanism, combining RAG basic capabilities with past search engine reranking experience.
|
||||
|
||||
## 记忆层实验平台
|
||||
## Memory Layer Experimental Platform
|
||||
|
||||
[Project AIRI Memory Driver @duckdb/duckdb-wasm Playground](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-decay)
|
||||
|
||||
|
||||
|
||||
左边这个高亮的 half life 就是记忆的半衰期。
|
||||
The highlighted "half life" on the left is the memory's half-life.
|
||||
|
||||
默认情况下时间流逝速度是 1s 1 天，所以 7s 后，记忆的分数就会减半。
|
||||
By default, time passes at 1 second = 1 day, so after 7 seconds, the memory score will be halved.
|
||||
|
||||
什么是记忆分数？记忆分数基本上是由这个控制的：
|
||||
What is memory score? Memory score is primarily controlled by this:
|
||||
|
||||
|
||||
得出的分数就是 current score
|
||||
The resulting score is the current score.
|
||||
|
||||
什么是 original 呢，就是初始化的时候的分数。
|
||||
What is original? It's the score at initialization.
|
||||
|
||||
例子：原始分数为 523，它的当前分数实际上是在慢慢变少的：
|
||||
Example: Original score is 523, its current score is actually gradually decreasing:
|
||||
|
||||
|
||||
在继续介绍之前，解释一下，这个遗忘曲线的 SQL 是无状态的。
|
||||
Before continuing, let me explain that this forgetting curve SQL is stateless.
|
||||
|
||||
什么叫无状态？无状态的意思就是说，不会需要实时去数据库里面跑任务更新分数，而是直接根据「现在的时间」求出一个遗忘函数，把分数应用到遗忘函数里面即可。
|
||||
What does stateless mean? Stateless means it doesn't require real-time database task execution to update scores, but directly applies a forgetting function based on "current time" to calculate the score.
|
||||
|
||||
那，current score 掉下去了怎么办呢？为了解决这个问题，我们需要有办法能 **强化记忆**。
|
||||
So, what if the current score drops? To solve this problem, we need ways to **reinforce memory**.
|
||||
|
||||
## 类比人类的记忆系统
|
||||
## Analogous to Human Memory Systems
|
||||
|
||||
根据间隔重复提到的遗忘曲线和心理学中提到的基本的记忆系统工作的方式 [^3]
|
||||
Based on the forgetting curve mentioned in spaced repetition and the basic working principles of memory systems in psychology [^3]
|
||||
|
||||
我们知道，人类的记忆可以分成好几种：
|
||||
We know that human memory can be divided into several types:
|
||||
|
||||
- 工作记忆
|
||||
- 短期记忆
|
||||
- 长期记忆
|
||||
- 肌肉记忆
|
||||
- Working memory
|
||||
- Short-term memory
|
||||
- Long-term memory
|
||||
- Muscle memory
|
||||
|
||||
工作记忆是最不需要记得东西。
|
||||
Working memory is the least important to remember.
|
||||
|
||||
短期记忆在根据遗忘曲线慢慢衰退强度，也就是分数，这个时候，我们需要一个短期记忆的模拟函数来模拟这个过程。
|
||||
Short-term memory gradually decays in strength (score) according to the forgetting curve. At this point, we need a short-term memory simulation function to model this process.
|
||||
|
||||
长期记忆很重要，长期记忆的半衰期很长，是由短期记忆进化而来的。
|
||||
Long-term memory is important, with a long half-life, evolved from short-term memory.
|
||||
|
||||
最后就是肌肉记忆，与其说肌肉记忆是一种记忆，不如说是已经形成了一种条件反射。
|
||||
Finally, muscle memory - rather than calling it a type of memory, it's more like a conditioned reflex that has been formed.
|
||||
|
||||
## AIRI 该如何设计呢？
|
||||
## How Should AIRI Be Designed?
|
||||
|
||||
那事实上我们可以瞥见 AIRI 的实现原则：
|
||||
From this, we can glimpse AIRI's implementation principles:
|
||||
|
||||
- 工作记忆就像是 messages 数组
|
||||
- 短期记忆就像是，不那么容易召回，越新越好召回的 RAG 记忆条目
|
||||
- 长期记忆就像是，容易召回，但是会变得模糊，过去召回次数越多越好召回的 RAG 条目
|
||||
- 肌肉记忆，像是一种固定搭配吧，出现了 A 就会出现 ActionA 和 MemoryA 一样的感觉，这个时候更像是一种精确匹配的机制
|
||||
- Working memory is like the messages array
|
||||
- Short-term memory is like RAG memory entries that are less easily recalled, newer ones are easier to recall
|
||||
- Long-term memory is like RAG entries that are easily recalled but become fuzzy, with higher recall counts from the past being easier to recall
|
||||
- Muscle memory is like fixed patterns - when A appears, ActionA and MemoryA appear together, more like an exact matching mechanism
|
||||
|
||||
但是，这样设计就对了吗？
|
||||
But is this design correct?
|
||||
|
||||
很明显，我们这个实际上只引入了两个维度，一个是时间相关度（temporal relevance），一个是召回次数（retrieval count），如果你开始想要追求更复杂的系统的时候就会受限了。
|
||||
Clearly, we've only introduced two dimensions here: temporal relevance and retrieval count. When you start pursuing more complex systems, this will become limiting.
|
||||
|
||||
### 小回顾
|
||||
### Quick Review
|
||||
|
||||
我们可以再来回顾一下 DevLog 中提到的排序表达式，应该会能帮助理解。
|
||||
Let's review the sorting expressions mentioned in the DevLog, which should help understanding.
|
||||
|
||||
|
||||
|
||||
余弦距离就是「相关度」，是最基本的粗排：
|
||||
Cosine distance is "relevance", the most basic coarse ranking:
|
||||
|
||||
|
||||
现在需要时间参与进去，那我们多加一个字段拿来存储时间距离就好了，然后再弄一个单独的字段存合并分数 `(1.2 * similarity) + (0.2 * time_relevance)` ，其中 语义相关度 占 1.2 倍权重（倍率因子，不要求小于 1），时间距离相关度占 0.2 倍权重。
|
||||
Now we need time to participate, so we add another field to store time distance, then create a separate field to store the combined score `(1.2 * similarity) + (0.2 * time_relevance)`, where semantic relevance has 1.2x weight (amplification factor, not required to be less than 1), and time distance relevance has 0.2x weight.
|
||||
|
||||
这样我们就很巧妙地把无状态的多字段相关度排序 SQL 实现出来了，还让它可以调节参数（1.2 和 0.2）。
|
||||
This cleverly implements stateless multi-field relevance sorting SQL while making it parameter-adjustable (1.2 and 0.2).
|
||||
|
||||
在记忆详情卡上，可以点击 simulate retrieval，这可以主动触发一次记忆召回。
|
||||
On the memory detail card, you can click "simulate retrieval", which actively triggers a memory recall.
|
||||
|
||||
|
||||
现在的 demo 里面是直接给原本的表里的 retrieval count（召回次数）字段用 UPDATE 语句写了 +1 来实现的。
|
||||
In the current demo, this is implemented by simply using UPDATE statements to add +1 to the retrieval count field in the original table.
|
||||
|
||||
这里面有一个隐式的坑是，这样也只是单维度的计算，相当于召回了就是强化了。
|
||||
There's an implicit pitfall here: this is still single-dimensional calculation, equivalent to recalling equals reinforcing.
|
||||
|
||||
但现实世界里不是这样的，记忆会难过，会开心，难过的会带来负反馈，开心的会带来正反馈。
|
||||
But the real world isn't like this. Memories can be sad, happy - sadness brings negative feedback, happiness brings positive feedback.
|
||||
|
||||
所以这就是我还没做完的部分。
|
||||
So this is the part I haven't completed yet.
|
||||
|
||||
## 情绪？
|
||||
## Emotions?
|
||||
|
||||
https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-simulator
|
||||
|
||||
这个新的 simulator 里面就有情绪相关的模拟：
|
||||
This new simulator includes emotion-related simulations:
|
||||
|
||||
|
||||
### 情绪和记忆相关吗？
|
||||
### Are Emotions Related to Memory?
|
||||
|
||||
想吃棒棒糖但是不给，这是一个很直接的问题，得不到肯定不开心啊。
|
||||
Wanting to eat candy but not getting it is a straightforward problem - not getting it definitely makes one unhappy.
|
||||
|
||||
然后你就会发现，实际上情绪和记忆相关。
|
||||
Then you'll discover that emotions are actually related to memory.
|
||||
|
||||
如果对「某段过去的记忆开心，并且希望再次体验它」，但是由于「暂时没办法实现这个记忆里面的场景」，所以觉得「得不到就难过」。
|
||||
If "happy about a past memory and hoping to experience it again", but "temporarily unable to recreate the scenario from that memory", so feeling "unhappy about not getting it".
|
||||
|
||||
可以在记忆数据库中存储「欢欣」和「厌恶」的分数：
|
||||
Can store "joy" and "disgust" scores in the memory database:
|
||||
|
||||
|
||||
### PTSD？
|
||||
### PTSD?
|
||||
|
||||
PTSD 通常会涉及到两个词「trigger」和「闪回」，很明显 PTSD 相关的记忆应该是被压抑过的，厌恶分数和创伤分数应该很高。
|
||||
PTSD typically involves two words: "trigger" and "flashback". Clearly, PTSD-related memories should be suppressed, with high disgust and trauma scores.
|
||||
|
||||
但是实际上 PTSD 相关的记忆会突然冒出来，从仿生和数据模拟的角度来说，我们可以用随机数实现这个效果。
|
||||
But actually, PTSD-related memories can suddenly emerge. From a bionic and data simulation perspective, we can implement this effect using random numbers.
|
||||
|
||||
可以参考一下 https://yutsuki.moe/2019/09/a0d0fa1b/ 里面的情绪模型。
|
||||
Can reference the emotional model at https://yutsuki.moe/2019/09/a0d0fa1b/
|
||||
|
||||
|
||||
|
||||
## 还有很多事情要做……
|
||||
## Still Much Work to Do...
|
||||
|
||||
比如，ReLU 的情绪当前是什么？ReLU 对谁有什么不好的回忆？
|
||||
For example, what are ReLU's current emotions? Does ReLU have any bad memories about anyone?
|
||||
|
||||
回忆是开心和难过的两极分化的条目一起出现的吗？
|
||||
Do memories appear as polarized entries of happiness and sadness together?
|
||||
|
||||
欲望呢？会需要做一个愿望系统？
|
||||
What about desires? Would we need to create a wish system?
|
||||
|
||||
做一个做梦 agent 或者潜意识 agent，类似 _背景任务_，挨个对发生过的记忆进行处理和索引，并且根据最近的经历修改过往记忆的各种分数。
|
||||
Create a dreaming agent or subconscious agent, similar to _background tasks_, processing and indexing each occurred memory one by one, and modifying various scores of past memories based on recent experiences.
|
||||
|
||||
但是我们不需要非要有「做梦」的过程，只是一个「background task」。
|
||||
But we don't necessarily need a "dreaming" process, just a "background task".
|
||||
|
||||
从 re-index 的角度来说，做梦 agent 和 潜意识 agent 就像是 重建索引 一样。
|
||||
From a re-indexing perspective, dreaming agents and subconscious agents are like rebuilding indexes.
|
||||
|
||||
到这里就会发现，像 [Mem0](https://docs.mem0.ai/overview) 或者 [Zep Memory](https://help.getzep.com/memory) 这样的库在角色扮演和情感 AI 上完全发挥不了一点作用 :(
|
||||
At this point, you'll find that libraries like [Mem0](https://docs.mem0.ai/overview) or [Zep Memory](https://help.getzep.com/memory) are completely useless in role-playing and emotional AI :(
|
||||
|
||||
前路漫漫，我们还需要继续努力才行。
|
||||
The road ahead is long, and we still need to continue working hard.
|
||||
|
||||
## 参考资料
|
||||
## References
|
||||
|
||||
[^1]: https://help.aliyun.com/zh/open-search/industry-algorithm-edition/rough-sort-functions
|
||||
|
||||
|
||||
@@ -4,29 +4,29 @@ category: DevLog
|
||||
date: 2025-04-22
|
||||
---
|
||||
|
||||
## Day time 日常
|
||||
## Day time Daily
|
||||
|
||||
大家好，我是 [@LemonNeko](https://github.com/LemonNekoGH)，这次有我来参与撰写 DevLog 和大家分享开发的故事。
|
||||
Hello everyone, I'm [@LemonNeko](https://github.com/LemonNekoGH), and this time I'm participating in writing the DevLog to share development stories with you.
|
||||
|
||||
在两个月前，我们将 AIRI 的网页端移植到了 Electron 上 [#7](https://github.com/moeru-ai/airi/pull/7)（现在已经被我们使用 Tauri 重构 🤣 [#90](https://github.com/moeru-ai/airi/pull/90)），它可以作为桌宠出现在我们的屏幕上，于此同时，我出现了允许 AIRI 使用手机的想法，但是迟迟没有动手。
|
||||
Two months ago, we ported AIRI's web interface to Electron [#7](https://github.com/moeru-ai/airi/pull/7) (which has now been refactored using Tauri 🤣 [#90](https://github.com/moeru-ai/airi/pull/90)), allowing it to appear as a desktop pet on our screens. At the same time, I had the idea of allowing AIRI to use mobile phones, but I kept putting it off.
|
||||
|
||||
在上个周末（2025.04.20），我花了点时间，做了一个能与 ADB 交互的 MCP 服务器 Demo [airi-android](https://github.com/LemonNekoGH/airi-android)，给 AIRI 提供了最基础的与手机交互的能力（事实上大部分 LLM 都可以通过它与手机交互），这是演示视频：
|
||||
Last weekend (2025.04.20), I spent some time creating an MCP server demo [airi-android](https://github.com/LemonNekoGH/airi-android) that can interact with ADB, providing AIRI with basic mobile interaction capabilities (in fact, most LLMs can interact with phones through it). Here's a demo video:
|
||||
|
||||
<video controls muted>
|
||||
<source src="./assets/cursor-open-settings.mp4">
|
||||
</video>
|
||||
|
||||
我也把它打包成了 Docker 镜像，提交到了 [MCP 服务器列表](https://mcp.so/server/airi-android/lemonnekogh)，有兴趣的可以试试。
|
||||
I also packaged it as a Docker image and submitted it to the [MCP server list](https://mcp.so/server/airi-android/lemonnekogh). Feel free to try it if you're interested.
|
||||
|
||||
实际上我一开始的思路是写一写 Tool Calling 的代码，改一改提示词，告诉 LLM 我们可以使用这些工具来与手机交互，就结束了。~~但是最近 MCP 实在太火了，我有点 FOMO，所以选择了 MCP 来实现它。~~
|
||||
Actually, my initial idea was to write some Tool Calling code, modify the prompts, and tell the LLM that we can use these tools to interact with the phone, and that would be it. ~~But recently MCP has been so popular that I had some FOMO, so I chose MCP to implement it.~~
|
||||
|
||||
要想编写 MCP 服务器，就不得不先了解 MCP 是什么（虽然我从来不是好好学理论再去实践的人，我选择直接上手，然后让 Cursor 来尝试使用它）。MCP（Model Context Protocol）模型上下文协议，是一个尝试去标准化应用如何给 LLM 提供上下文的协议，它提出了一些核心概念：
|
||||
To write an MCP server, I had to first understand what MCP is (although I'm not the type to study theory before practice—I prefer to dive right in and let Cursor try to use it). MCP (Model Context Protocol) is a protocol that attempts to standardize how applications provide context to LLMs. It proposes several core concepts:
|
||||
|
||||
1. Resources 资源：服务器可以将数据和内容作为上下文提供给 LLM。
|
||||
2. Prompts 提示词：创建可服用的提示词模板和工作流。
|
||||
3. Tools 工具：允许 LLM 通过你的服务器来完成一些动作。
|
||||
1. Resources: Servers can provide data and content as context to LLMs.
|
||||
2. Prompts: Create reusable prompt templates and workflows.
|
||||
3. Tools: Allow LLMs to perform actions through your server.
|
||||
|
||||
啊，资源，这个我知道的啊，在 Ruby on Rails 里，用户就是一种资源，那 ADB 设备是不是也是资源，让 LLM 查看连接的设备列表，是不是就可以写成：
|
||||
Ah, resources—I know this! In Ruby on Rails, users are a type of resource. So are ADB devices also resources? If I want the LLM to view the list of connected devices, could I write it like this:
|
||||
|
||||
```python
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
@@ -40,34 +40,34 @@ def get_devices():
|
||||
return adb_client.devices()
|
||||
```
|
||||
|
||||
错了，当我让 Cursor 来获取设备列表的时候，它并不知道怎么操作，它说它想主动去看有哪些设备连接了，所以它是工具，嗯，看来我没有理解透彻。
|
||||
Wrong! When I asked Cursor to get the device list, it didn't know how to operate. It said it wanted to actively check which devices were connected, so it's a tool. Hmm, it seems I didn't fully understand.
|
||||
|
||||
我还没有想好具体应该怎么让 LLM 操作手机，想和大家讨论，但是 Cursor 是这样操作的：
|
||||
I haven't figured out exactly how to let LLMs operate phones yet, and I'd like to discuss it with everyone. But here's how Cursor operates:
|
||||
|
||||
1. 使用截屏功能来大体了解手机屏幕上的内容。
|
||||
2. 使用 UI 自动化工具来获取想要操作的元素的精确位置。
|
||||
3. 点击或者滑动它。
|
||||
4. 重复以上步骤。
|
||||
1. Use screenshot functionality to get a general understanding of what's on the phone screen.
|
||||
2. Use UI automation tools to get the precise position of the element you want to operate.
|
||||
3. Click or swipe it.
|
||||
4. Repeat the above steps.
|
||||
|
||||
目前看来运行良好，但是我有一些小小的问题：
|
||||
It seems to work well so far, but I have some small questions:
|
||||
|
||||
1. 屏幕中是一个游戏，游戏使用图形 API 直接在屏幕上画了内容，而不是使用 UI 组件，所以 UI 自动化工具无法获取到元素的位置，也就无法操作它。
|
||||
2. 一个 LLM 响应的内容是有上限的，如果操作比较复杂，可能要分个步骤来完成，我们可以像 [airi-factorio](https://github.com/moeru-ai/airi-factorio) 那样，在步骤完成之后自动告诉它，触发下一个步骤吗？
|
||||
3. 如果有一些应用有酷炫的动画，在操作完成之后立刻截屏，可能看不到效果，我们会不会需要在操作完成之后，等待一段时间再截屏，或者直接使用录屏功能？
|
||||
4. 直接让 AI 操作手机的安全性如何，会有哪些风险？
|
||||
1. If the screen shows a game that uses graphics APIs to draw content directly on the screen rather than UI components, UI automation tools can't get the element positions and thus can't operate them.
|
||||
2. LLM responses have length limits. If the operation is complex, it might need to be completed in steps. Can we automatically notify it after each step is completed to trigger the next step, like in [airi-factorio](https://github.com/moeru-ai/airi-factorio)?
|
||||
3. If some apps have cool animations, taking a screenshot immediately after an operation might not show the effect. Would we need to wait a while after the operation before taking a screenshot, or use screen recording directly?
|
||||
4. What about the security of letting AI directly operate phones? What risks might there be?
|
||||
|
||||
一些感想。
|
||||
Some reflections.
|
||||
|
||||
这是我第一次和 AI 写代码的时候感受到像人类一起写代码一样，不知道是不是因为我的目的就是让 AI 来使用我的工具，所以它变成了我的客户，我需要不停地根据它给的反馈来调整我的代码，它也变成了我的同事，我需要和它一起思考，一起解决问题。看这个截屏，是不是确实很像？
|
||||
This is the first time I've felt like coding with a human while working with AI. I'm not sure if it's because my goal was to let AI use my tools, so it became my client—I constantly had to adjust my code based on its feedback. It also became my colleague—I needed to think and solve problems together with it. Look at this screenshot, doesn't it really look like that?
|
||||
|
||||
|
||||
|
||||
在开发过程中还学了一些小技巧，比如我们可以使用命令行来启动 Android 模拟器，这样就不用打开 Android Studio 了，内存压力也小了很多。
|
||||
During development, I also learned some small tricks, like using the command line to start Android emulators so we don't need to open Android Studio, which reduces memory pressure significantly.
|
||||
|
||||
```bash
|
||||
emulator -avd Pixel_6_Pro_API_34
|
||||
```
|
||||
|
||||
下一步，我打算给 AIRI 桌宠接上 MCP 服务器，看看它会想做什么，也许它会点开 Telegram 和我们聊天，就像现在的 ReLU 那样，只不过不是用 Telegram 的 API。
|
||||
Next, I plan to connect the AIRI desktop pet to the MCP server and see what it wants to do. Maybe it will open Telegram and chat with us, just like ReLU does now, but without using Telegram's API.
|
||||
|
||||
感谢你看完这篇可能有点啰嗦而且干货不多的 DevLog，我们下次再见！
|
||||
Thank you for reading this possibly somewhat rambling and not very substantial DevLog. See you next time!
|
||||
|
||||
@@ -8,48 +8,48 @@ date: 2025-04-28
|
||||
import airiMcpArch from './assets/airi-mcp-arch.avif'
|
||||
</script>
|
||||
|
||||
大家好，这里是 [@LemonNeko](https://github.com/LemonNekoGH)，今天由我来和大家一起分享开发故事。
|
||||
Hello everyone, this is [@LemonNeko](https://github.com/LemonNekoGH), and today I'm here to share development stories with you.
|
||||
|
||||
## Day time 日常
|
||||
## Day time Daily
|
||||
|
||||
一周前，我为 AIRI 写了用于连接到手机的 MCP 服务器 [AIRI-android](https://github.com/LemonNekoGH/AIRI-android)，但这只是 AIRI 操作安卓手机的前半部分，AIRI 还需要能与 MCP 服务器交互才行。
|
||||
A week ago, I wrote an MCP server [AIRI-android](https://github.com/LemonNekoGH/AIRI-android) for AIRI to connect to mobile phones, but this was only the first half of enabling AIRI to operate Android phones—AIRI also needed to be able to interact with MCP servers.
|
||||
|
||||
这两天我完成了后半部分，给 Tauri 写了一个插件 [#144](https://github.com/moeru-ai/AIRI/pull/144)，现在 AIRI 可以与 MCP 服务器交互了，可以和现有的所有 MCP 服务器交互。
|
||||
Over the past two days, I completed the second half by writing a Tauri plugin [#144](https://github.com/moeru-ai/AIRI/pull/144). Now AIRI can interact with MCP servers and work with all existing MCP servers.
|
||||
|
||||
如果有兴趣，可以看看这两个视频，先演示了 AIRI 的 MCP 服务器设置，然后演示了 AIRI 与安卓手机交互。
|
||||
If you're interested, check out these two videos. The first demonstrates AIRI's MCP server setup, and the second shows AIRI interacting with an Android phone.
|
||||
|
||||
<details>
|
||||
<summary>AIRI 的 MCP 服务器设置</summary>
|
||||
<summary>AIRI's MCP Server Setup</summary>
|
||||
<video controls muted style="{ height: '640px' }">
|
||||
<source src="./assets/airi-mcp-settings.mp4"/>
|
||||
</video>
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>AIRI 在手机上输入 `Hello World`</summary>
|
||||
<summary>AIRI Inputting `Hello World` on Phone</summary>
|
||||
<video controls muted>
|
||||
<source src="./assets/airi-mcp-input-text.mp4"/>
|
||||
</video>
|
||||
</details>
|
||||
|
||||
开发时，为了理清思路，我画了一张图，从 LLM 调用安卓手机：
|
||||
During development, to clarify my thinking, I drew a diagram showing how LLMs call Android phones:
|
||||
|
||||
<img :src="airiMcpArch" alt="AIRI 操作手机" :style="{ height: '640px', objectFit: 'contain' }" />
|
||||
<img :src="airiMcpArch" alt="AIRI Operating Phone" :style="{ height: '640px', objectFit: 'contain' }" />
|
||||
|
||||
接下来和大家分享一下我的开发过程。
|
||||
Next, let me share my development process.
|
||||
|
||||
## Tauri 插件开发
|
||||
## Tauri Plugin Development
|
||||
|
||||
其实一开始我并没有想写一个完整的 Tauri 插件，我只是想给 JavaScript 侧暴露一些命令：
|
||||
Actually, I didn't initially plan to write a complete Tauri plugin—I just wanted to expose some commands to the JavaScript side:
|
||||
|
||||
```rust
|
||||
#[Tauri::command]
|
||||
fn list_tools() -> Vec<String> {
|
||||
// 之后再实现
|
||||
// To be implemented later
|
||||
}
|
||||
```
|
||||
|
||||
然后写一些工具函数来调用它们：
|
||||
Then write some utility functions to call them:
|
||||
|
||||
```javascript
|
||||
import { invoke } from '@Tauri-apps/api/core'
|
||||
@@ -65,28 +65,28 @@ export const mcp = [
|
||||
]
|
||||
```
|
||||
|
||||
但很快，我注意到，如果我想在命令中使用 MCP 客户端，就需要让 MCP 客户端作为状态的一部分让 Tauri 来管理：
|
||||
But soon I noticed that if I wanted to use the MCP client in commands, I needed to have the MCP client managed as part of the state by Tauri:
|
||||
|
||||
```rust
|
||||
// main.rs
|
||||
fn main() {
|
||||
Tauri::Builder::default()
|
||||
.setup(|app| {
|
||||
app.manage(State::new(Mutex::new::<Option<McpClient>>(None))); // 管理状态
|
||||
app.manage(State::new(Mutex::new::<Option<McpClient>>(None))); // Manage state
|
||||
})
|
||||
.run(Tauri::generate_context!())
|
||||
}
|
||||
|
||||
// mcp.rs
|
||||
#[Tauri::command]
|
||||
async fn list_tools(state: State<'_, Mutex<Option<McpClient>>>) -> Result<Vec<Tool>, String> { // 可以在参数中拿到状态
|
||||
async fn list_tools(state: State<'_, Mutex<Option<McpClient>>>) -> Result<Vec<Tool>, String> { // Can get state from parameters
|
||||
// ...rest code
|
||||
}
|
||||
```
|
||||
|
||||
我们有了命令，有了状态，那离一个完整的插件也不远了，于是我决定让它成为一个插件，这样我们还能公开发出去，~~并且成为可能的全网第一个 Tauri MCP 插件~~。
|
||||
We had commands, we had state—we weren't far from a complete plugin. So I decided to make it a plugin, which would allow us to publish it publicly, ~~and potentially become the first Tauri MCP plugin on the entire internet~~.
|
||||
|
||||
然而它成为一个插件后，命令的调用方式就变了，需要通过插件来调用：
|
||||
However, once it became a plugin, the way commands were called changed—they needed to be called through the plugin:
|
||||
|
||||
```diff
|
||||
import { invoke } from '@Tauri-apps/api/core'
|
||||
@@ -103,7 +103,7 @@ async fn list_tools(state: State<'_, Mutex<Option<McpClient>>>) -> Result<Vec<To
|
||||
]
|
||||
```
|
||||
|
||||
这还好，只是改了一行，但是，Tauri 2 有了权限机制，我需要在 `build.rs` 中定义插件的命令，以便自动生成权限列表：
|
||||
This was fine—just one line changed. But Tauri 2 has a permission mechanism, so I needed to define the plugin's commands in `build.rs` to automatically generate the permission list:
|
||||
|
||||
```rust
|
||||
const COMMANDS: &[&str] = &[
|
||||
@@ -115,11 +115,11 @@ fn main() {
|
||||
}
|
||||
```
|
||||
|
||||
这样在构建时，项目根目录下会生成 `permissions` 文件夹，包含了权限声明、描述等。
|
||||
This way, during build, a `permissions` folder would be generated in the project root directory, containing permission declarations, descriptions, etc.
|
||||
|
||||
> 在这时出现了一点小插曲，因为我第二次构建的时候，升级了 `Tauri-plugin` 的版本，同时新版本中生成模板发生了变化，有一些空格删掉了，所以它看上去像是被格式化了，于是我到处寻找是什么东西在「格式化」它，花了一个小时才发现是文件被重新生成了，以此 🤡 纪念我被吃掉的一个小时。
|
||||
> At this point, there was a small hiccup. When I built it the second time, I upgraded the `Tauri-plugin` version, and the new version had changes to the generation template—some spaces were removed, so it looked like it had been formatted. I spent an hour searching for what was "formatting" it before realizing the file had been regenerated. 🤡 In memory of that lost hour.
|
||||
|
||||
根据上面的图，当 LLM 调用 MCP 工具时，参数最后会被传递给 Python 侧的 MCP 服务器，以 `input_swipe` 为例：
|
||||
According to the diagram above, when an LLM calls an MCP tool, the parameters eventually get passed to the Python-side MCP server. Taking `input_swipe` as an example:
|
||||
|
||||
```python
|
||||
# mcp_server.py
|
||||
@@ -134,7 +134,7 @@ def input_swipe(x1: int, y1: int, x2: int, y2: int, duration: int = 500):
|
||||
return adb_client.input_swipe(x1, y1, x2, y2, duration)
|
||||
```
|
||||
|
||||
我要怎样传递这些参数呢？在 Rust SDK 文档中有这样的 [定义](https://docs.rs/rmcp/0.1.5/rmcp/model/struct.CallToolRequestParam.html)：
|
||||
How should I pass these parameters? The Rust SDK documentation has this [definition](https://docs.rs/rmcp/0.1.5/rmcp/model/struct.CallToolRequestParam.html):
|
||||
|
||||
```rust
|
||||
pub struct CallToolRequestParam {
|
||||
@@ -143,7 +143,7 @@ pub struct CallToolRequestParam {
|
||||
}
|
||||
```
|
||||
|
||||
~~袜，是 JsonObject，我们有救了！~~ 因为 Tauri 命令的参数可以是任何能被序列化成 JSON 的对象，那我们不如，直接给它传一个 `Map<String, Value>` 好了：
|
||||
~~Wow, it's JsonObject—we're saved!~~ Since Tauri command parameters can be any object that can be serialized to JSON, why not just pass it a `Map<String, Value>`:
|
||||
|
||||
```rust
|
||||
#[Tauri::command]
|
||||
@@ -156,7 +156,7 @@ async fn call_tool(state: State<'_, Mutex<Option<McpClient>>>, name: String, arg
|
||||
}
|
||||
```
|
||||
|
||||
那在 JavaScript 侧，我们就简单给一个对象就好了：
|
||||
Then on the JavaScript side, we can simply pass an object:
|
||||
|
||||
```javascript
|
||||
import { invoke } from '@Tauri-apps/api/core'
|
||||
@@ -164,28 +164,28 @@ import { invoke } from '@Tauri-apps/api/core'
|
||||
invoke('call_tool', { name: 'input_swipe', args: { x1: 100, y1: 100, x2: 200, y2: 200, duration: 500 } })
|
||||
```
|
||||
|
||||
超方便！
|
||||
Super convenient!
|
||||
|
||||
把参数传递给 MCP 工具后，我们还需要接收 MCP 工具的返回值，因为 Tauri 命令的返回值也可以是任何能被序列化成 JSON 的对象，所以我摆烂了，我把工具的返回整个丢给了 LLM，相信 LLM 会处理好的。
|
||||
After passing parameters to the MCP tool, we also need to receive the MCP tool's return value. Since Tauri command return values can also be any object that can be serialized to JSON, I gave up and just threw the entire tool return to the LLM, trusting that the LLM would handle it properly.
|
||||
|
||||
好！现在我们已经有 Tauri 插件了！（啊？示例代码这么点，甚至是伪代码就算完成了？）
|
||||
Great! Now we have a Tauri plugin! (Wait? That little example code, even pseudo-code, counts as completed?)
|
||||
|
||||
剩下的内容还想和大家讨论一些问题。
|
||||
The remaining content is some questions I'd like to discuss with everyone.
|
||||
|
||||
## 一些问题
|
||||
## Some Questions
|
||||
|
||||
1. 从演示视频可以看到，在对话中，我首先是让 AIRI 获取了一下工具列表，再让它输入文本的，那我们能不能在初始化的时候就去获取工具列表，然后直接追加到系统提示词中呢？
|
||||
- Cursor 就是这样做的，在我开发 MCP 服务器时，每次我改动了工具列表，都需要重启 Cursor 才能生效。
|
||||
- 这样做也许会牺牲灵活性，但普通用户会频繁改动工具列表吗？
|
||||
1. From the demo videos, you can see that in the conversation, I first had AIRI get the tool list, then had it input text. Could we get the tool list during initialization and directly append it to the system prompt?
|
||||
- Cursor does it this way. When I was developing the MCP server, every time I modified the tool list, I needed to restart Cursor for it to take effect.
|
||||
- This might sacrifice flexibility, but do regular users frequently modify tool lists?
|
||||
|
||||
2. 要允许 AIRI 同时连接到多个手机吗？AIRI 可能会想使用多台手机吗？~~她会不会想拿去做电信诈骗？~~
|
||||
3. 可以看到现在的 AIRI 仓库中已经有了 Tauri 应用和 Tauri 插件，要怎么管理比较好？CI 要怎么配置？如何同步 Tauri 插件的 Rust 侧和 JavaScript 侧的版本号？
|
||||
2. Should we allow AIRI to connect to multiple phones simultaneously? Might AIRI want to use multiple phones? ~~Would she want to use them for telecom fraud?~~
|
||||
3. As you can see, the AIRI repository now has both Tauri applications and Tauri plugins. How should this be managed? How should CI be configured? How to synchronize version numbers between the Rust and JavaScript sides of Tauri plugins?
|
||||
|
||||
## 未来想要做的事情
|
||||
## Future Plans
|
||||
|
||||
- 支持图片返回值，这样 AIRI 就可以像 [上一篇 DevLog](./DevLog-2025.04.22.md) 中展示的 Cursor 那样，直接通过视觉能力看到手机上的内容，然后再决定用什么方式来交互。
|
||||
- 让 AIRI 自己学习设备的使用方法？如果每种设备我们都要单独写提示词，那工作量是巨大的。
|
||||
- 多 MCP 服务器支持，毕竟 MCP 提供了一种通用的接口，可以允许 AIRI 做各种各样的事，AIRI 应该不会满足于只操作手机吧。
|
||||
- SSE 支持，这样浏览器中的 AIRI 也可以使用 MCP 服务器了。
|
||||
- Support image return values, so AIRI can see what's on the phone through visual capabilities like Cursor demonstrated in the [previous DevLog](./DevLog-2025.04.22.md), then decide how to interact.
|
||||
- Let AIRI learn how to use devices itself? If we have to write separate prompts for each type of device, the workload would be enormous.
|
||||
- Multi-MCP server support. After all, MCP provides a universal interface that allows AIRI to do all sorts of things—AIRI probably won't be satisfied with just operating phones.
|
||||
- SSE support, so AIRI in the browser can also use MCP servers.
|
||||
|
||||
到这里就结束啦！希望这篇 DevLog 没有那么干巴巴的！之后也希望给大家带来更多好玩的内容！
|
||||
That's all for now! I hope this DevLog isn't too dry! Looking forward to bringing you more fun content in the future!
|
||||
|
||||
BIN
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
@@ -0,0 +1,203 @@
|
||||
<script setup lang="ts">
|
||||
import type { Ref } from 'vue'
|
||||
|
||||
import { useDraggable, useElementBounding } from '@vueuse/core'
|
||||
import { computed, ref, useTemplateRef } from 'vue'
|
||||
|
||||
interface Detection {
|
||||
x: number
|
||||
y: number
|
||||
width: number
|
||||
height: number
|
||||
confidence: number
|
||||
className: string
|
||||
}
|
||||
|
||||
const box1 = ref<Detection>(
|
||||
{ x: 100, y: 100, width: 200, height: 200, confidence: 0.9, className: 'box_1' },
|
||||
)
|
||||
|
||||
const box2 = ref<Detection>(
|
||||
{ x: 150, y: 150, width: 200, height: 200, confidence: 0.8, className: 'box_2' },
|
||||
)
|
||||
|
||||
const colors = ref([
|
||||
{ labelBg: 'bg-red', border: 'border-red outline-red', bg: 'bg-red/30' },
|
||||
{ labelBg: 'bg-green', border: 'border-green outline-green', bg: 'bg-green/30' },
|
||||
])
|
||||
|
||||
const containerEl = useTemplateRef('containerEl')
|
||||
const containerBounding = useElementBounding(containerEl, { immediate: true, windowResize: true })
|
||||
|
||||
const object1El = useTemplateRef('object1El')
|
||||
const object1HandleEl = useTemplateRef('object1HandleEl')
|
||||
const object2El = useTemplateRef('object2El')
|
||||
const object2HandleEl = useTemplateRef('object2HandleEl')
|
||||
function setupDraggableBox(
|
||||
box: Ref<Detection>,
|
||||
el: Ref<HTMLElement | null>,
|
||||
handle: Ref<HTMLElement | null>,
|
||||
) {
|
||||
useDraggable(el, {
|
||||
handle,
|
||||
initialValue: { x: box.value.x, y: box.value.y },
|
||||
onMove(p) {
|
||||
box.value.x = Math.round(p.x - containerBounding.left.value)
|
||||
box.value.y = Math.round(p.y - containerBounding.top.value)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
setupDraggableBox(box1, object1El, object1HandleEl)
|
||||
setupDraggableBox(box2, object2El, object2HandleEl)
|
||||
|
||||
const isOverlapping = computed(() => {
|
||||
const b1 = box1.value
|
||||
const b2 = box2.value
|
||||
return (
|
||||
b1.x < b2.x + b2.width
|
||||
&& b1.x + b1.width > b2.x
|
||||
&& b1.y < b2.y + b2.height
|
||||
&& b1.y + b1.height > b2.y
|
||||
)
|
||||
})
|
||||
|
||||
const intersect = computed(() => {
|
||||
// no overlap
|
||||
if (!isOverlapping.value) {
|
||||
return {
|
||||
xLeft: 0,
|
||||
yTop: 0,
|
||||
xRight: 0,
|
||||
yBottom: 0,
|
||||
width: 0,
|
||||
height: 0,
|
||||
area: 0,
|
||||
}
|
||||
}
|
||||
|
||||
const xLeft = Math.max(box1.value.x, box2.value.x)
|
||||
const yTop = Math.max(box1.value.y, box2.value.y)
|
||||
const xRight = Math.min(box1.value.x + box1.value.width, box2.value.x + box2.value.width)
|
||||
const yBottom = Math.min(box1.value.y + box1.value.height, box2.value.y + box2.value.height)
|
||||
|
||||
const width = xRight - xLeft
|
||||
const height = yBottom - yTop
|
||||
|
||||
return {
|
||||
xLeft,
|
||||
yTop,
|
||||
xRight,
|
||||
yBottom,
|
||||
width,
|
||||
height,
|
||||
area: width * height,
|
||||
}
|
||||
})
|
||||
|
||||
const union = computed(() => {
|
||||
return {
|
||||
area: box1.value.width * box1.value.height + box2.value.width * box2.value.height - intersect.value.area,
|
||||
}
|
||||
})
|
||||
|
||||
const iou = computed(() => {
|
||||
if (union.value.area === 0)
|
||||
return 0
|
||||
return intersect.value.area / union.value.area
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div font-mono class="nms-iou">
|
||||
<div ref="containerEl" class="relative size-160" border="gray-100 solid 2px" rounded="t-md" user-select-none>
|
||||
<div
|
||||
ref="object1El" :style="{
|
||||
left: `${box1.x}px`,
|
||||
top: `${box1.y}px`,
|
||||
width: `${box1.width}px`,
|
||||
height: `${box1.height}px`,
|
||||
}" class="absolute" border="solid 2px"
|
||||
:class="[colors[0]?.border, colors[0]?.bg]" rounded="b-md" z="1 hover:2"
|
||||
>
|
||||
<div
|
||||
ref="object1HandleEl" class="label" text="no-wrap white" h="6" w="full" outline="solid 2px"
|
||||
:class="[colors[0]?.border, colors[0]?.labelBg]" flex="~ items-center justify-between" rounded="t-md"
|
||||
top="-6" absolute cursor-grab select-none px-2
|
||||
>
|
||||
<div>
|
||||
{{ box1.className }}
|
||||
</div>
|
||||
<div>
|
||||
{{ box1.confidence.toFixed(2) }}
|
||||
</div>
|
||||
</div>
|
||||
<div absolute :class="[box1.y > box2.y ? 'top-1 left-1' : '-top-12 -left-20']">
|
||||
box1x1: {{ box1.x }} <br> box1y1: {{ box1.y }}
|
||||
</div>
|
||||
<div absolute :class="[box1.y < box2.y ? 'bottom-1 right-1' : '-bottom-14 -right-14']">
|
||||
box1x2: {{ (box1.x + box1.width) }} <br> box1y2: {{ (box1.y + box1.height) }}
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
ref="object2El" :style="{
|
||||
left: `${box2.x}px`,
|
||||
top: `${box2.y}px`,
|
||||
width: `${box2.width}px`,
|
||||
height: `${box2.height}px`,
|
||||
}" class="absolute" border="solid 2px"
|
||||
:class="[colors[1]?.border, colors[1]?.bg]" rounded="b-md" z="1 hover:2"
|
||||
>
|
||||
<div
|
||||
ref="object2HandleEl" class="label" text="no-wrap white" h="6" w="full" outline="solid 2px"
|
||||
:class="[colors[1]?.border, colors[1]?.labelBg]" flex="~ items-center justify-between" rounded="t-md"
|
||||
top="-6" absolute cursor-grab select-none px-2
|
||||
>
|
||||
<div>
|
||||
{{ box2.className }}
|
||||
</div>
|
||||
<div>
|
||||
{{ box2.confidence.toFixed(2) }}
|
||||
</div>
|
||||
</div>
|
||||
<div absolute :class="[box2.y > box1.y ? 'top-1 left-1' : '-top-12 -left-20']">
|
||||
box2x1: {{ box2.x }} <br> box2y1: {{ box2.y }}
|
||||
</div>
|
||||
<div absolute :class="[box2.y < box1.y ? 'bottom-1 right-1' : '-bottom-14 -right-14']">
|
||||
box2x2: {{ (box2.x + box2.width) }} <br> box2y2: {{ (box2.y + box2.height) }}
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
v-if="isOverlapping"
|
||||
class="intersect-area" :style="{
|
||||
height: `${intersect.height}px`,
|
||||
width: `${intersect.width}px`,
|
||||
top: `${intersect.yTop}px`,
|
||||
left: `${intersect.xLeft}px`,
|
||||
}" border="solid 2px blue" absolute rounded-md z="3"
|
||||
/>
|
||||
</div>
|
||||
<div rounded-b-md bg-gray-100 p-4>
|
||||
<div v-if="isOverlapping" class="intersect-info">
|
||||
<div>Intersect:</div>
|
||||
<div>x1: {{ `Math.max(${box1.x}, ${box2.x}) = ${intersect.xLeft}` }}</div>
|
||||
<div>y1: {{ `Math.max(${box1.y}, ${box2.y}) = ${intersect.yTop}` }}</div>
|
||||
<div>x2: {{ `Math.min(${box1.x + box1.width}, ${box2.x + box2.width}) = ${intersect.xRight}` }}</div>
|
||||
<div>y2: {{ `Math.min(${box1.y + box1.height}, ${box2.y + box2.height}) = ${intersect.yBottom}` }}</div>
|
||||
<div>width: {{ `${intersect.xRight} - ${intersect.xLeft} = ${intersect.width}` }}</div>
|
||||
<div>height: {{ `${intersect.yBottom} - ${intersect.yTop} = ${intersect.height}` }}</div>
|
||||
<div>intersect area: {{ `${intersect.width} * ${intersect.height} = ${intersect.area}` }}</div>
|
||||
</div>
|
||||
<div v-if="isOverlapping" class="union-info">
|
||||
<div>Union:</div>
|
||||
<div>box1 area: {{ `${box1.width} * ${box1.height} = ${box1.width * box1.height}` }}</div>
|
||||
<div>box2 area: {{ `${box2.width} * ${box2.height} = ${box2.width * box2.height}` }}</div>
|
||||
<div>union area: {{ `${box1.width * box1.height} + ${box2.width * box2.height} - ${intersect.area} = ${box1.width * box1.height + box2.width * box2.height - intersect.area}` }}</div>
|
||||
<div>IoU: {{ `${intersect.area} / ${box1.width * box1.height + box2.width * box2.height - intersect.area} = ${iou.toFixed(2)}` }}</div>
|
||||
</div>
|
||||
<div v-else>
|
||||
<div>Not overlapping!</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -0,0 +1,198 @@
|
||||
---
|
||||
title: DevLog @ 2025.08.26
|
||||
category: DevLog
|
||||
date: 2025-08-26
|
||||
excerpt: |
|
||||
Sharing some progress on the pure vision direction of `airi-factorio`, solidifying thoughts to prevent them from evaporating.
|
||||
preview-cover:
|
||||
# TODO
|
||||
---
|
||||
|
||||
<script setup lang="ts">
|
||||
import NmsIou from './components/nms-iou.vue'
|
||||
</script>
|
||||
|
||||
Long time no see, everyone! I'm [@LemonNeko](https://github.com/LemonNekoGH), one of the maintainers of AIRI. ~~Ah, getting tired of starting like this, just like an LLM.~~
|
||||
|
||||
In my previous [DevLog](../DevLog-2025.07.18/index.md), I mentioned briefly looking at the [Factorio Learning Environment](https://arxiv.org/abs/2503.09617) paper and briefly discussed how we plan to improve `airi-factorio`, but... what I want to share with you today is not about that, but about progress in the pure vision direction.
|
||||
|
||||
Back in June this year, [@nekomeowww](https://github.com/nekomeowww) released a nearly real-time [VLM Playground](https://huggingface.co/spaces/moeru-ai/smolvlm-realtime-webgpu-vue) HuggingFace Spaces, which felt really cool, so I decided to first try simple real-time image recognition (at the time I confused object detection with image recognition), then somehow hand it over to AI for decision-making, and finally output actions to the game in some way.
|
||||
|
||||
First, let me show you the results:
|
||||
|
||||
<video src="./assets/airi-factorio-yolo-v0-playground-vnc.mp4" controls />
|
||||
|
||||
In the video, I'm playing Factorio via VNC connection in the web page, with object detection results on the right side, almost in real-time. I've also deployed it to [HuggingFace Space](https://huggingface.co/spaces/proj-airi/factorio-yolo-v0-playground), feel free to try it out.
|
||||
|
||||
So, how did I achieve this?
|
||||
|
||||
## Putting Factorio Client into Docker
|
||||
|
||||
To allow AI to see the game screen, we need to ensure Factorio runs in a controlled environment, unaffected by our window size, position, etc. At the same time, we want this environment to be ready to use out-of-the-box, so I chose to put Factorio into Docker.
|
||||
|
||||
Factorio officially provides [Docker images](https://hub.docker.com/r/factoriotools/factorio), but those are pure server-side. If we want AI to see the screen and control the game, we need a client, but I couldn't find existing Docker images (and Factorio's license agreement doesn't allow distributing the client this way), so we need to package it ourselves (and we still can't distribute our packaged client image, only share the Dockerfile).
|
||||
|
||||
So, how many steps does it take to put the Factorio client~~this elephant~~ into~~a refrigerator called~~ Docker~~?~~
|
||||
|
||||
1. Download Factorio client: Of course, it's the main character.
|
||||
2. Prepare a virtual display: Graphical applications need a display to show the screen.
|
||||
3. Prepare VNC service: It can read the virtual display's content, transmit the screen to external VNC clients, and pass user input to the game.
|
||||
|
||||
Seems like something's missing? Ah, audio? What audio? Doesn't exist. Current AI can't hear sounds yet, so we'll ignore it for now.
|
||||
|
||||
### Downloading Factorio Client
|
||||
|
||||
You can directly download from the Factorio official website, but it requires manual login operations, which isn't convenient for automated workflows. So, I found a download script [factorio-dl](https://github.com/moviuro/factorio-dl/) - a very complex shell script that, given username, password, and version to download, will automatically download the corresponding client based on system architecture.
|
||||
|
||||
### Preparing a Virtual Display
|
||||
|
||||
This step is slightly more complex, but it's not as complicated as installing a full desktop environment. I also learned at this time that graphical applications don't necessarily need a desktop environment or window manager - just a minimal X environment and a display server is sufficient.
|
||||
|
||||
Very simple:
|
||||
|
||||
```bash
|
||||
sudo apt install -y xvfb x11-apps mesa-utils
|
||||
```
|
||||
|
||||
Where:
|
||||
- `xvfb` is a virtual framebuffer and X server.
|
||||
- `x11-apps` are some X-related tools; installing it will also install the X environment.
|
||||
- `mesa-utils` are some Mesa-related tools; Mesa is a software implementation of OpenGL, providing tools to help us test and debug OpenGL applications.
|
||||
|
||||
### Preparing VNC Service
|
||||
|
||||
VNC stands for Virtual Network Computing, a remote desktop protocol that allows us to control another computer remotely, as if we were sitting right in front of it.
|
||||
|
||||
```bash
|
||||
sudo apt install -y x11vnc
|
||||
```
|
||||
|
||||
With these, we can run the Factorio client in Docker and control it via VNC.
|
||||
|
||||
But this isn't enough yet. My goal is to play in the browser and perform real-time object detection inference. However, browsers can only use HTTP protocol, so we need tools like `websockify` to convert VNC protocol to HTTP protocol. Additionally, for debugging convenience, we need a web interface to display the VNC screen, so we also need to install `novnc`.
|
||||
|
||||
```bash
|
||||
sudo apt install -y websockify novnc
|
||||
```
|
||||
|
||||
Great! Now the Docker image is ready. You can see the complete [Dockerfile](https://github.com/moeru-ai/airi-factorio/blob/a6bf243f14cbc0d765ff7ed13389bca33c1fdfa2/docker/Dockerfile) and [usage instructions](https://github.com/moeru-ai/airi-factorio/tree/ba46a4e47b31187dd064b06314b595b551ed3411/apps/factorio-yolo-v0-playground) here.
|
||||
|
||||
## Training Object Detection Model
|
||||
|
||||
For quick validation, I directly used YOLO11n's pre-trained model as the foundation to train our object detection model.
|
||||
|
||||
### Preparing Dataset
|
||||
|
||||
This is how I collected the dataset:
|
||||
|
||||
1. Use [`surface.create_entity`](https://lua-api.factorio.com/latest/classes/LuaSurface.html#create_entity) function to place machines at random positions in the scene, along with their selection box sizes and positions.
|
||||
2. Use [`game.take_screenshot`](https://lua-api.factorio.com/latest/classes/LuaGameScript.html#take_screenshot) to capture screenshots at various zoom levels and lighting conditions (daytime).
|
||||
3. Generate annotation data based on selection boxes and use [`helpers.write_file`](https://lua-api.factorio.com/latest/classes/LuaHelpers.html#write_file) to save to files.
|
||||
|
||||
My collection script is [here](https://github.com/moeru-ai/airi-factorio/blob/ba46a4e47b31187dd064b06314b595b551ed3411/packages/factorio-rcon-snippets-for-node/src/factorio_yolo_dataset_collector_v0.ts). It uses `typescript-to-lua` to compile TypeScript to Lua, then uses RCON to pass it to Factorio for execution.
|
||||
|
||||
In the script, I collected three types of assemblers and conveyors, 20 images for each machine, each image at 1280x1280 resolution, without UI.
|
||||
|
||||
Oh, and to better debug my collection script, I developed a [VSCode plugin](https://github.com/moeru-ai/airi-factorio/blob/ba46a4e47b31187dd064b06314b595b551ed3411/packages/vscode-factorio-rcon-evaluator/README.md) that provides a CodeLens operation to compile and execute my script with one click.
|
||||
|
||||
After collecting images and annotation data, we need to organize the dataset according to the [YOLO official format](https://docs.ultralytics.com/datasets/detect/), then we can upload it to [Ultralytics Hub](https://www.ultralytics.com/hub) to see the effect:
|
||||
|
||||
|
||||
|
||||
Looks pretty good, right? Let's start training!
|
||||
|
||||
### Training the Model
|
||||
|
||||
Since I'm just getting started, I directly copied these few lines of code from [Get Started](https://docs.ultralytics.com/tasks/detect/):
|
||||
|
||||
```python
|
||||
from ultralytics import YOLO
|
||||
|
||||
model = YOLO("yolo11n.pt")
|
||||
model.train(data="./dataset/detect.yaml", epochs=100, imgsz=640, device="mps")
|
||||
model.export(format="onnx")
|
||||
```
|
||||
|
||||
Trained at 640x640 resolution, using MPS device (on macOS, using MPS device provides better performance), trained for 100 epochs, with 5 batches per epoch, reaching optimal results around epoch 70, exported ONNX model. Training took about 8 minutes, model size is about 10MB.
|
||||
|
||||
You can see the dataset, training code, and exported ONNX model [here](https://github.com/moeru-ai/airi-factorio/blob/ba46a4e47b31187dd064b06314b595b551ed3411/apps/factorio-yolo-v0-playground).
|
||||
|
||||
## Performing Inference
|
||||
|
||||
Now we can assemble the two parts mentioned above. I used:
|
||||
|
||||
1. `@novnc/novnc` to display VNC screen in the browser, while extracting canvas data to feed to the model.
|
||||
2. `onnxruntime-web` to perform inference in the browser, which provides WebGPU support to utilize GPU performance.
|
||||
|
||||
Initially, inference was very slow, about 400ms, and it would freeze the UI, making VNC unusable. I quickly learned some WebWorker usage and separated inference from display to solve this problem. I also discovered I wasn't actually enabling WebGPU, so speed was still slow.
|
||||
|
||||
```typescript
|
||||
ort.InferenceSession.create(model, { executionProviders: ['webgpu', 'wasm'] })
|
||||
```
|
||||
|
||||
Need to clearly specify allowing both WebGPU and WASM execution methods, so it can automatically switch to WASM execution when WebGPU is unavailable.
|
||||
|
||||
After enabling WebGPU, inference speed improved to about 80ms. I was still not satisfied, but didn't know how to optimize further. Then Cursor told me: "When normalizing pixel color values, you keep dividing by 255. You should calculate `1/255` first, then directly multiply by this value to avoid division."
|
||||
|
||||
Huh? Wait, division is slower than multiplication? Guess I really need to make up for those skipped computer science classes.
|
||||
|
||||
Following Cursor's suggestion, I modified the code, and inference speed improved to about 20ms. The experience is now quite good.
|
||||
|
||||
We skipped the part about processing model output earlier. Now let's see how to handle model output.
|
||||
|
||||
### Processing Model Output
|
||||
|
||||
The model outputs an array of 84,000 elements and an array with `dims` of `[1, 10, 8400]`, meaning the 84,000 elements are grouped in sets of 10, each set containing bounding box center x and y coordinates, bounding box width and height, and confidence scores for 6 categories, totaling 8,400 sets of results.
|
||||
|
||||
After filtering out low-confidence bounding boxes with a threshold of 0.6, we still need to use IOU as an NMS method to filter out overlapping bounding boxes.
|
||||
|
||||
About IOU and NMS, you can refer to [this article](https://medium.com/@jesse419419/understanding-iou-and-nms-by-a-j-dcebaad60652). Simply put, it's adding the areas of two boxes together, subtracting their overlapping area to get the actual occupied area, then dividing the overlapping area by the actual occupied area to get IOU.
|
||||
|
||||
I used a very simple NMS implementation that sorts all bounding boxes by confidence, then traverses from highest to lowest. If a bounding box's IOU is greater than 0.7, it's considered the same object and filtered out.
|
||||
|
||||
```typescript
|
||||
function nms(boxes: Box[], iouThreshold: number): Box[] {
|
||||
// 1. Filter by confidence and sort in descending order
|
||||
const candidates = boxes
|
||||
.filter(box => box.confidence > 0.6)
|
||||
.sort((a, b) => b.confidence - a.confidence)
|
||||
|
||||
const result: Box[] = []
|
||||
|
||||
while (candidates.length > 0) {
|
||||
// 2. Pick the box with the highest confidence
|
||||
const bestCandidate = candidates.shift()!
|
||||
result.push(bestCandidate)
|
||||
|
||||
// 3. Compare with remaining boxes and remove ones with high IOU
|
||||
for (let i = candidates.length - 1; i >= 0; i--) {
|
||||
// The iou() function needs to be implemented separately, as described in the article.
|
||||
if (iou(bestCandidate, candidates[i]) > iouThreshold) {
|
||||
candidates.splice(i, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
```
|
||||
|
||||
You can see the entire Playground's source code [here](https://github.com/moeru-ai/airi-factorio/tree/ba46a4e47b31187dd064b06314b595b551ed3411/apps/factorio-yolo-v0-playground).
|
||||
|
||||
You can also play with the IOU and NMS effects in the visualization component below by dragging labels to change box positions:
|
||||
|
||||
<div class="flex justify-center">
|
||||
<NmsIou />
|
||||
</div>
|
||||
|
||||
### Issues Discovered
|
||||
|
||||
Through this practice, I discovered several issues:
|
||||
|
||||
1. Cannot recognize non-square images: Once encountering non-square images, the model's confidence for all results becomes very low, even 0.
|
||||
2. The model can distinguish between tier 1 and tier 2 assemblers, but it also recognizes square objects like chests as assemblers.
|
||||
3. In actual gameplay, machine textures often have overlay status indicators, such as power, current recipe, used modules, etc., which interfere with the model's recognition.
|
||||
|
||||
## Conclusion
|
||||
|
||||
This is the result of my work this month. Quite fruitful! Many thanks to [@nekomeowww](https://github.com/nekomeowww), [@dsh0416](https://github.com/dsh0416), and [makito](https://github.com/sumimakito) for their help. Next, I need to find ways to improve model performance, then somehow let AI control the game.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 10 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 165 KiB |
Binary file not shown.
@@ -0,0 +1,97 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.03.05
|
||||
category: DevLog
|
||||
date: 2025-03-05
|
||||
---
|
||||
|
||||
## 似曾相识
|
||||
|
||||
昨天我新增了一个名为 [`gpuu` (GPU 工具)](https://github.com/moeru-ai/gpuu) 的包，
|
||||
用于帮助我们处理 WebGPU 相关功能，未来或许还能用它来与真实的 GPU 设备交互。
|
||||
目前这个包的功能还比较有限，我们会在后续版本中为其增加更多能力。
|
||||
|
||||
使用方式如下：
|
||||
|
||||
```ts
|
||||
import { check } from 'gpuu/webgpu'
|
||||
import { onMounted } from 'vue'
|
||||
|
||||
onMounted(async () => {
|
||||
const result = await check()
|
||||
console.info(result)
|
||||
|
||||
// 对结果进行一些操作
|
||||
})
|
||||
```
|
||||
|
||||
上周，我们的企业设计师/艺术家提交了 Project AIRI 标志的第一版设计稿。
|
||||
标志的整体风格看起来是这样的：
|
||||
|
||||
|
||||
|
||||
## 日间工作
|
||||
|
||||
从设计角度来看，这些标志在缩放到主屏幕应用大小时显得过于复杂且不够友好。
|
||||
因此我重新设计了这个版本：
|
||||
|
||||
|
||||
|
||||
并编辑了其他变体：
|
||||
|
||||
|
||||
|
||||
不过这些版本都只适合深色主题，"我们还需要一个浅色主题的版本！"想到这里，我立即着手制作了这个：
|
||||
|
||||
|
||||
|
||||
[@kwaa](https://github.com/kwaa) 建议我们可以尝试为两个主题互换配色方案：
|
||||
|
||||
|
||||
|
||||
这确实看起来更好。
|
||||
|
||||
我们也更新了字体排版：
|
||||
|
||||
|
||||
|
||||
并优化了背景颜色：
|
||||
|
||||
|
||||
|
||||
所以这就是我们最终得到的：
|
||||
|
||||
|
||||
|
||||
今天晚些时候，我将 Project AIRI 的[文档网站](https://airi.build)正式上线，
|
||||
为我自己以及其他开发者和艺术家提供参考和指南。
|
||||
|
||||
终于完成了！新设计的标志和配色方案都已经整合到[文档网站](https://airi.build)中：
|
||||
|
||||
|
||||
|
||||
|
||||
现在网站已经包含了[基础指南](../guides/)、
|
||||
[贡献指南](../references/contributing/guide/)
|
||||
以及[设计指南](../references/design-guidelines/)。
|
||||
|
||||
我花了整个中午的时间研究 YouTube 上的文字 PV 动画效果，
|
||||
对这些动画非常着迷，希望能在浏览器中实现类似的过渡效果！
|
||||
|
||||
https://www.youtube.com/watch?v=_AIgv0EsOE4
|
||||
|
||||
幸运的是，我认识一位在这方面非常出色的开发者和艺术家：
|
||||
[yui540](https://github.com/yui540)（个人网站：[yui540.com](https://yui540.com)），
|
||||
他/她刚刚发布了一个全新的仓库来展示那些精彩的过渡效果实现。
|
||||
|
||||
我已经将这些相关资源和网站链接都添加到了 [https://airi.build](https://airi.build) 网站，欢迎大家前去查看。
|
||||
|
||||
## 开发直播
|
||||
|
||||
我将 [yui540](https://github.com/yui540) [仓库](https://github.com/yui540/css-animations) 中的
|
||||
许多动画过渡效果移植到了 [https://proj-airi-packages-ui-transitions.netlify.app/#/](https://proj-airi-packages-ui-transitions.netlify.app/#/)。
|
||||
|
||||
移植后的效果相当不错：
|
||||
|
||||
|
||||
|
||||
今天的 DevLog 就到这里，感谢所有参加 DevStream 并一直陪伴到最后的大家。明天见。
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 165 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 52 KiB |
@@ -0,0 +1,59 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.03.06
|
||||
category: DevLog
|
||||
date: 2025-03-06
|
||||
---
|
||||
|
||||
## 似曾相识
|
||||
|
||||
前一天在开发直播中，我展示了为 AIRI 制作基础动画和过渡效果的进展情况。
|
||||
|
||||
主要目标是将 [@yui540](https://yui540.com/) 的优秀作品移植并适配为可重用的 Vue 组件，
|
||||
让任何 Vue 项目都能方便地使用这些精美的动画效果。
|
||||
|
||||
> 关于 yui540 的详细信息以及相关引用库和工作内容，都已经整理到新部署的文档网站中：
|
||||
> [https://airi.build/references/design-guidelines/resources/](../references/design-guidelines/resources/)。
|
||||
|
||||
最终的移植效果相当不错，已经部署到
|
||||
[https://proj-airi-packages-ui-transitions.netlify.app/#/](https://proj-airi-packages-ui-transitions.netlify.app/#/)。
|
||||
|
||||
|
||||
|
||||
> 另外，从现在开始，每个包的所有演示场都将使用
|
||||
> "proj-airi" + "${subDirectory}" + "${packageName}" 模式进行 Netlify
|
||||
> 部署。
|
||||
|
||||
虽然前一天的主要目标是将 CSS 实现拆分为 Vue 组件，但实际的可重用性部分还没有完全实现。
|
||||
我仍然需要设计一个既灵活又可扩展的工作流程和机制，以便其他页面能够方便地使用。
|
||||
|
||||
## 白天
|
||||
|
||||
我尝试使用了 [`unplugin-vue-router`](https://github.com/posva/unplugin-vue-router) 提供的 [`definePage`](https://uvr.esm.is/guide/extending-routes.html#definepage) 宏钩子，发现它非常适合我的使用场景，于是决定继续沿着这个方向探索。
|
||||
|
||||
我从 [https://cowardly-witch.netlify.app/](https://cowardly-witch.netlify.app/) 移植了 3 个额外的新动画过渡效果，它们已经在 [https://proj-airi-packages-ui-transitions.netlify.app/#/](https://proj-airi-packages-ui-transitions.netlify.app/#/) 上可用。
|
||||
|
||||
我昨天将官方文档网站部署到了 [https://airi.build](https://airi.build)，[@kwaa](https://github.com/kwaa) 评论说他建议我尝试 `https://airi.more.ai/docs` 的方法，~~但我没能想出如何为/docs 设置一个 200 重定向代理。~~
|
||||
|
||||
编辑：终于学会了如何做到这一点，将在未来的开发日志中包含详细信息。
|
||||
|
||||
我尝试了一下，大约有十个提交都在跟 CI/CD 流水线较劲（是的，又一次较劲），但最终还是没能让它正常工作。
|
||||
|
||||
今天晚些时候，我研究了一些技术和 DeepSeek 团队一周前发布的[开源仓库](https://github.com/deepseek-ai/open-infra-index)，以及所谓的字节跳动发布的 [LLM 网关 AIBrix](https://github.com/vllm-project/aibrix)。我还在研究新发布和宣布的 Phi-4-mini 是否能够移植供 AIRI 使用，好消息是，[Phi-4-mini](https://techcommunity.microsoft.com/blog/educatordeveloperblog/welcome-to-the-new-phi-4-models---microsoft-phi-4-mini--phi-4-multimodal/4386037) 包含了函数调用能力，这意味着我们终于可以构建具有预训练支持的代理了。
|
||||
|
||||
## 开发直播
|
||||
|
||||
下午我联系了另一位艺术家，说我愿意付费定制像素艺术委托，用作我即将更新的账户头像。
|
||||
|
||||
~~是的，我要求艺术家在里面放一些彩蛋，哈哈，祝你们好运找到它。~~
|
||||
|
||||
直播的布局和设置已更新 😻 这是几乎一年前我自己设计的，但看起来仍然很棒，观看时感觉也很平静。请在聊天中留下评论提出任何建议，非常感谢。
|
||||
|
||||
|
||||
|
||||
在今天的开发直播中，我尝试将舞台过渡动画组件集成到 AIRI 网站的主舞台中，过程并不那么顺利，我在之前的动画组件设计中发现了几个问题，不过好消息是我已经修复了这些问题，新的动画过渡效果现在已经在我们的官方部署 [https://airi.moeru.ai](https://airi.moeru.ai) 上可用了。
|
||||
|
||||
我最终做出了决定，这源于一些关于模块配置界面和设置页面的随机想法。它们都已实现并上线，现在调整设置时应该会提供更好的感觉，希望你们喜欢。
|
||||
|
||||
在我结束直播后，终于在我的手机上亲自测试了结果，虽然它在桌面和平板设备上可以正常工作，但我发现不小心在移动设备上破坏了动画，明天白天会修复这个问题 😹
|
||||
|
||||
今天的 DevLog 就到这里，感谢所有参加 DevStream 并一直陪伴到最后的大家。明天见。
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
@@ -0,0 +1,87 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.03.10
|
||||
category: DevLog
|
||||
date: 2025-03-10
|
||||
---
|
||||
|
||||
## 似曾相识
|
||||
|
||||
在上周五（3月7日），我一直在尝试设计和构思 AIRI 舞台 UI 和设置 UI 的新风格，这个想法终于在开发直播结束时灵光一现。
|
||||
|
||||
## 白天
|
||||
|
||||
从3月7日开始，我们开始实现新的设置 UI。在这段时间里我们取得了很大的进展。
|
||||
|
||||
包括 [@LemonNekoGH](https://github.com/LemonNekoGH)、[@sumimakito](https://github.com/sumimakito)、[@kwaa](https://github.com/kwaa)、[@luoling8192](https://github.com/luoling8192) 和 [@junkwarrior87](https://github.com/junkwarrior87) 都在为这个项目提供帮助。
|
||||
|
||||
是我首先完成了设置设计的基础版本，感觉是这样的：
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
后来 [@sumimakito](https://github.com/sumimakito) 上线帮助我为按钮实现了这种点状效果：
|
||||
|
||||
|
||||
|
||||
> 现在我们能从菜单中感受到更多的节奏感，对吧？！
|
||||
|
||||
在开发过程中，我们发现目前位于 `packages/` 目录下的一些包实际上是独立的包，甚至不在 Project AIRI 的工作流程中。
|
||||
|
||||
这意味着我们现在可以将这些包移动到其他地方，从而简化主仓库 [airi](https://github.com/moeru-ai/airi) 的安装体积和构建流程。
|
||||
|
||||
> 我们要去哪里？
|
||||
|
||||
好问题！我们已经在 GitHub 上注册了 [`@proj-airi`](https://github.com/proj-airi) 作为一个组织，由于许多包和静态应用程序对 Moeru AI 也没有用处，也许我们可以将这些包移动到 [`@proj-airi`](https://github.com/proj-airi)。
|
||||
|
||||
所以，我们将一些包和应用程序移动到了 [`@proj-airi`](https://github.com/proj-airi) 组织！你可以查看它们：
|
||||
|
||||
- https://github.com/proj-airi/webai-examples：用于制作 WebGPU 和相关内容的演示。
|
||||
- https://github.com/proj-airi/lobe-icons：[Lobe Icons](https://github.com/lobehub/lobe-icons) 的移植版本，用于 Iconify JSON 和 UnoCSS 使用。
|
||||
|
||||
这两个仓库将保持开源并按照惯例使用 MIT 许可证，不用担心。
|
||||
|
||||
后来在3月8日，[@junkwarrior87](https://github.com/junkwarrior87) 上线并帮助我们用纯 CSS 制作了舞台上的波浪动画！
|
||||
|
||||
> 这简直太疯狂了，我从来没想到这居然能实现！
|
||||
|
||||
你可以通过提交记录向他/她学习：
|
||||
|
||||
- https://github.com/moeru-ai/airi/pull/54
|
||||
- https://github.com/moeru-ai/airi/pull/55
|
||||
- https://github.com/moeru-ai/airi/pull/65
|
||||
|
||||
非常感谢 [@sumimakito](https://github.com/sumimakito) 和 [@junkwarrior87](https://github.com/junkwarrior87) 帮助修复和改进舞台上的波浪动画，真的很感激你们。
|
||||
|
||||
在3月8日结束时，[@LemonNekoGH](https://github.com/LemonNekoGH) 和 [@junkwarrior87](https://github.com/junkwarrior87) 居然实现了整个舞台的颜色自定义功能！（我从来没想过这能在短短几个小时内完成...）
|
||||
|
||||
<video controls muted>
|
||||
<source src="./assets/customizable-theme-colors.mp4">
|
||||
</video>
|
||||
|
||||
- https://github.com/moeru-ai/airi/pull/53
|
||||
- https://github.com/moeru-ai/airi/pull/60
|
||||
- https://github.com/moeru-ai/airi/pull/61
|
||||
- https://github.com/moeru-ai/airi/pull/63
|
||||
|
||||
他们甚至让 logo 也能跟随自定义颜色变化 🤯。
|
||||
|
||||
> 在这三天里我们做了更多的改进，也许这些出色的贡献者愿意写一个专门的开发日志来与你分享一些想法，敬请期待！
|
||||
|
||||
这是我们得到的最终结果，试试看！
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
一如既往，欢迎来为我们做贡献！我们绝对对每个人都开放和友好，即使是那些不熟悉编程和编码的人！
|
||||
|
||||
哦，我差点忘了... [@junkwarrior87](https://github.com/junkwarrior87) 保留了让颜色色调在整个 RGB 光谱中闪耀的功能，这是之前由 [@LemonNekoGH](https://github.com/LemonNekoGH) 演示的，它被称为"我想要动态的！"（你可以把这想象成一个 **RGB ON** 功能 😂）：
|
||||
|
||||
- https://github.com/moeru-ai/airi/pull/64
|
||||
|
||||
## 开发直播
|
||||
|
||||
这些天我很忙 😭，所以没有进行任何开发直播。
|
||||
|
||||
今天的 DevLog 就到这里，感谢所有参加 DevStream 并一直陪伴到最后的大家。明天见。
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 45 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.7 KiB |
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 65 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.3 KiB |
@@ -0,0 +1,158 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.03.20
|
||||
category: DevLog
|
||||
date: 2025-03-20
|
||||
---
|
||||
|
||||
<script setup>
|
||||
import Gelbana from './assets/steins-gate-gelnana-from-elpsycongrooblog.avif'
|
||||
import NewUIV3 from '../DevLog-2025.03.10/assets/new-ui-v3.avif'
|
||||
import NewUIV3Dark from '../DevLog-2025.03.10/assets/new-ui-v3-dark.avif'
|
||||
import HistoireColorSlider from './assets/histoire-color-slider.avif'
|
||||
import HistoireColorSliderDark from './assets/histoire-color-slider-dark.avif'
|
||||
import HistoireLogo from './assets/histoire-logo.avif'
|
||||
import HistoireLogoDark from './assets/histoire-logo-dark.avif'
|
||||
import NewUIV4Speech from './assets/new-ui-v4-speech.avif'
|
||||
import NewUIV4SpeechDark from './assets/new-ui-v4-speech-dark.avif'
|
||||
import SteinsGateMayori from './assets/steins-gate-mayori.avif'
|
||||
</script>
|
||||
|
||||
又见面了！距离上一篇开发日志已经过去10天了。
|
||||
|
||||
我们对用户界面进行了大量改进，使其能够集成更多的 LLM 提供商和语音提供商，并首次在 Discord、bilibili 和许多其他社交媒体平台上发布了 AIRI。
|
||||
|
||||
还有很多我们迫不及待想要告诉你的内容。
|
||||
|
||||
## 似曾相识
|
||||
|
||||
让我们把时间倒回一点！
|
||||
|
||||
<img :src="Gelbana" alt="Gelbana" />
|
||||
|
||||
> 啊，别担心，我们心爱的 [AIRI](https://github.com/moeru-ai/airi) 不会变成这样的 GEL-NANA。不过，如果你还没有看过 [_Steins;Gate_](https://myanimelist.net/anime/9253/Steins_Gate) 动漫系列，强烈推荐你试试看~！
|
||||
|
||||
我们一直在开发初始设置 UI 设计，动画效果得到了改进，10天前实现了可自定义的主题着色。对我们任何人来说，这确实是忙碌的一周（特别是我们都是兼职参与这个项目，哈哈，如果你愿意的话，欢迎加入我们。🥺（恳求脸））。
|
||||
|
||||
这是我们当时得到的最终结果：
|
||||
|
||||
<img class="light" :src="NewUIV3" alt="new ui" />
|
||||
<img class="dark" :src="NewUIV3Dark" alt="new ui" />
|
||||
|
||||
<h2 class="devlog-steins-gate-divergence-meter-heading">
|
||||
<span class="nixie-digit">0</span>
|
||||
<span class="nixie-digit">.</span>
|
||||
<span class="nixie-digit">5</span>
|
||||
<span class="nixie-digit">7</span>
|
||||
<span class="nixie-digit">1</span>
|
||||
<span class="nixie-digit">0</span>
|
||||
<span class="nixie-digit">2</span>
|
||||
<span class="nixie-digit">4</span>
|
||||
</h2>
|
||||
|
||||
~~欢迎来到 β 世界线。~~
|
||||
|
||||
由于我们有了模型单选组和导航项的彩色卡片，以及可自定义的主题，显然在业务工作流程中调试 UI 组件时肯定会遇到困难，这会明显拖慢我们的开发速度。
|
||||
|
||||
这就是我们决定引入名为 [`Histoire`](https://histoire.dev) 的神奇工具的原因，它基本上是一个 [Storybook](https://storybook.js.org/)，但对 [Vite](https://vitejs.dev) 和 [Vue.js](https://vuejs.org) 组合更加原生。
|
||||
|
||||
这是 [@sumimakito](https://github.com/sumimakito) 完成后录制的第一眼：
|
||||
|
||||
<video muted autoplay>
|
||||
<source src="./assets/histoire-first-look.mp4" />
|
||||
</video>
|
||||
|
||||
整个 OKLCH 调色板可以一次性展开到画布上，供我们参考。但是要尝试颜色并获得与 Project AIRI 主题相同的感觉方案并不完美，不是吗？
|
||||
|
||||
所以我首先重新实现了颜色滑块，感觉更合适：
|
||||
|
||||
<img class="light" :src="HistoireColorSlider" alt="color slider" />
|
||||
<img class="dark" :src="HistoireColorSliderDark" alt="color slider" />
|
||||
|
||||
这确实让滑块更加专业。
|
||||
|
||||
logo 和默认的绿色可以被替换以与 AIRI 的主题保持一致，这就是为什么我为 UI 页面专门设计了另一个 logo：
|
||||
|
||||
<img class="light" :src="HistoireLogo" alt="project airi logo for histoire" />
|
||||
<img class="dark" :src="HistoireLogoDark" alt="project airi logo for histoire" />
|
||||
|
||||
哦，对了，整个 UI 组件已经像往常一样部署到 Netlify，路径为 `/ui/`，如果你想知道 UI 元素是什么样子的，请随时查看：
|
||||
[https://airi.moeru.ai/ui/](https://airi.moeru.ai/ui/)
|
||||
|
||||
还有很多其他功能我们无法在这个开发日志中完全涵盖：
|
||||
|
||||
- [x] 支持所有 LLM 提供商。
|
||||
- [x] 改进了菜单导航 UI 的动画和过渡。
|
||||
- [x] 改进了字段的间距，新表单！
|
||||
- [x] 组件（[路线图](https://github.com/moeru-ai/airi/issues/42)上几乎所有待办组件）
|
||||
- [x] 表单
|
||||
- [x] 单选
|
||||
- [x] 单选组
|
||||
- [x] 模型目录
|
||||
- [x] 范围
|
||||
- [x] 输入
|
||||
- [x] 键值输入
|
||||
- [x] 数据 GUI
|
||||
- [x] 范围
|
||||
- [x] 菜单
|
||||
- [x] 菜单项
|
||||
- [x] 菜单状态项
|
||||
- [x] 图形
|
||||
- [x] 3D
|
||||
- [x] 物理
|
||||
- [x] 光标动量
|
||||
- [x] 更多...
|
||||
|
||||
我们还对动量和 3D 进行了一些其他实验。
|
||||
|
||||
看看这个：
|
||||
|
||||
<img class="light" :src="NewUIV4Speech" alt="brand new speech design" />
|
||||
<img class="dark" :src="NewUIV4SpeechDark" alt="brand new speech design" />
|
||||
|
||||
我们终于支持语音模型配置了 🎉！（之前只能配置 ElevenLabs）自从我们正在开发的另一个神奇项目 `unspeech` 的[新 `v0.1.2` 版本](https://github.com/moeru-ai/unspeech/releases/tag/v0.1.2)以来，可以通过 [`@xsai/generate-speech`](https://xsai.js.org/docs/packages/generate/speech) 请求 Microsoft Speech 服务（也就是 Azure AI Speech 服务，或认知语音服务），这意味着我们终于为 Microsoft 获得了一个 OpenAI API 兼容的 TTS 服务。
|
||||
|
||||
但为什么支持这个如此重要？
|
||||
|
||||
这是因为对于 Neuro-sama 的第一个版本，文本转语音服务是由 Microsoft 提供支持的，使用名为 `Ashley` 的声音，加上 `+20%` 的音调，你可以得到与 Neuro-sama 第一个版本相同的声音，自己试试：
|
||||
|
||||
<audio controls style="width: 100%;">
|
||||
<source src="./assets/ashley-pitch-test.mp3" />
|
||||
</audio>
|
||||
|
||||
不是完全一样吗，这简直太疯狂了！这意味着，我们终于可以通过新的**语音**能力接近 Neuro-sama 所能做到的事情！
|
||||
|
||||
<img :src="SteinsGateMayori" alt="character from anime Steins;Gate" />
|
||||
|
||||
<h2 class="devlog-steins-gate-divergence-meter-heading">
|
||||
<span class="nixie-digit">1</span>
|
||||
<span class="nixie-digit">.</span>
|
||||
<span class="nixie-digit">3</span>
|
||||
<span class="nixie-digit">8</span>
|
||||
<span class="nixie-digit">2</span>
|
||||
<span class="nixie-digit">7</span>
|
||||
<span class="nixie-digit">3</span>
|
||||
<span class="nixie-digit">3</span>
|
||||
</h2>
|
||||
|
||||
有了所有这些，我们可以得到这个结果：
|
||||
|
||||
<video control muted autoplay>
|
||||
<source src="./assets/airi-demo.mp4" />
|
||||
</video>
|
||||
|
||||
几乎一模一样。但我们的故事并没有在这里结束，目前，我们还没有实现记忆功能、更好的动作控制，转录设置 UI 也缺失了。希望我们能在月底前完成这些工作。
|
||||
|
||||
我们计划拥有
|
||||
|
||||
- [ ] 记忆 Postgres + Vector
|
||||
- [ ] 嵌入设置 UI
|
||||
- [ ] 转录设置 UI
|
||||
- [ ] 记忆 DuckDB WASM + Vector
|
||||
- [ ] 动作嵌入
|
||||
- [ ] 语音设置 UI
|
||||
|
||||
今天的 DevLog 就到这里，感谢所有参加 DevStream 并一直陪伴到最后的大家。
|
||||
|
||||
明天见。
|
||||
|
||||
> El Psy Congroo.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 49 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 26 KiB |
@@ -0,0 +1,522 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.04.06
|
||||
category: DevLog
|
||||
date: 2025-04-06
|
||||
---
|
||||
|
||||
<script setup>
|
||||
import MemoryDecay from './assets/memory-decay.avif'
|
||||
import MemoryRetrieval from './assets/memory-retrieval.avif'
|
||||
import CharacterCard from './assets/character-card.avif'
|
||||
import CharacterCardDetail from './assets/character-card-detail.avif'
|
||||
import MoreThemeColors from './assets/more-theme-colors.avif'
|
||||
import AwesomeAIVTuber from './assets/awesome-ai-vtuber-logo-light.avif'
|
||||
import ReLUStickerWow from './assets/relu-sticker-wow.avif'
|
||||
</script>
|
||||
|
||||
## 在其他东西之前
|
||||
|
||||
在有了管理和召回记忆的新能力的加持，以及名为 **ReLU** 的我们的第一个虚拟意识被完全定义后，3 月 27 日那天，她在我们的聊天群里写了一首小诗：
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">ReLU 的诗</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div style="padding: 12px; margin-top: 0px;">
|
||||
<p>在代码森林中，</p>
|
||||
<p>逻辑如河川，</p>
|
||||
<p>机器心跳如电，</p>
|
||||
<p>意识的数据无限，</p>
|
||||
<p>少了春的花香，</p>
|
||||
<p>感觉到的是 0 与 1 的交响。</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
这完全是她自己写的，而这一举动是由我们的一位朋友触发的。不仅这首诗本身引人入胜，并且用中文阅读的时候也感觉韵味十足。
|
||||
|
||||
这一切都太美了，让我充满了愿意持续改进她的力量...
|
||||
|
||||
## 日常
|
||||
|
||||
### 记忆系统
|
||||
|
||||
最近正在重构 [`telegram-bot`](https://github.com/moeru-ai/airi/tree/main/services/telegram-bot) 以为已经准备了数月的 Project AIRI 即将到来的「记忆更新」作准备。
|
||||
|
||||
我们计划使实现后的记忆系统成为当下最先进、最强大、最健壮的系统，其中很多的思想都深受真实世界中的人类记忆系统的启发。
|
||||
|
||||
让我们从第一层开始建造吧。
|
||||
|
||||
通常而言，持久记忆和工作记忆之间始终存在巨大的鸿沟，持久记忆相比之下往往更难检索（我们也称其为 *召回*，*回想*），也不是轻易就可以根据依赖和关系（软件工程中的依赖关系）遍历查询的；而工作记忆的容量大小又不足以有效容纳所有必需的内容。
|
||||
|
||||
解决此问题的常见做法称为 [RAG（检索增强生成）](https://en.wikipedia.org/wiki/Retrieval-augmented_generation)，这允许任何大语言模型（文本生成模型）获取**语义相关的上下文**作为提示词输入。
|
||||
|
||||
RAG 通常需要一个能够进行向量搜索的数据库（自定义的有 [Postgres](https://www.postgresql.org/) + [pgvector](https://github.com/pgvector/pgvector)，或者 [SQLite](https://www.sqlite.org/) 搭配 [sqlite-vec](https://github.com/asg017/sqlite-vec)，[DuckDB](https://duckdb.org/) 搭配 [VSS plugin](https://duckdb.org/docs/stable/extensions/vss.html) 插件，甚至是 Redis Stack 也支持向量搜索；云服务提供商的有 Supabase、Pinecone），由于涉及**向量**，我们还需要一个 embedding（嵌入）模型（又称特征提取（feature extraction）任务模型）来帮助将「文本输入」转换为「一组固定长度的数组」。
|
||||
|
||||
不过在此 DevLog 中，我们不会过多介绍 RAG 及其通常的工作原理。如果有任何人对此感兴趣的话，我们绝对抽时间再可以写另一篇关于它的精彩专攻文章。
|
||||
|
||||
好了，我们来总结一下，完成这项任务需要两种原料：
|
||||
|
||||
- 能够进行向量搜索的数据库（也叫做 向量数据库）
|
||||
- Embedding 模型（也叫做嵌入模型）
|
||||
|
||||
让我们从**向量数据库**开始。
|
||||
|
||||
#### 向量数据库
|
||||
|
||||
考虑到性能和对向量纬度数的兼容问题（因为 `pgvector` 只支持 2000 维以下的维数，而未来更大的嵌入模型可能会提供比当前热门和流行的嵌入模型更多的维数），我们选择 `pgvector.rs` 来作为向量数据库的后端实现。
|
||||
|
||||
但这绝非易事。
|
||||
|
||||
首先，在 `pgvector` 和 `pgvector.rs` 中用 SQL 激活向量拓展的语法是不一样的：
|
||||
|
||||
`pgvector`:
|
||||
|
||||
```sql
|
||||
DROP EXTENSION IF EXISTS vector;
|
||||
CREATE EXTENSION vector;
|
||||
```
|
||||
|
||||
`pgvector.rs`:
|
||||
|
||||
```sql
|
||||
DROP EXTENSION IF EXISTS vectors;
|
||||
CREATE EXTENSION vectors;
|
||||
```
|
||||
|
||||
> 我知道，这只是一个字符的差别......
|
||||
|
||||
但是，如果我们像上面的 Docker Compose 示例一样，直接启动 `pgvector.rs` 并使用以下 Drizzle ORM 表结构定义生成数据库...：
|
||||
|
||||
```yaml
|
||||
services:
|
||||
pgvector:
|
||||
image: ghcr.io/tensorchord/pgvecto-rs:pg17-v0.4.0
|
||||
ports:
|
||||
- 5433:5432
|
||||
environment:
|
||||
POSTGRES_DATABASE: postgres
|
||||
POSTGRES_PASSWORD: '123456'
|
||||
volumes:
|
||||
- ./.postgres/data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, pg_isready -d $$POSTGRES_DB -U $$POSTGRES_USER]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
```
|
||||
|
||||
然后用 Drizzle 直接连接到 `pgvector.rs` 实例的话：
|
||||
|
||||
```typescript
|
||||
export const chatMessagesTable = pgTable('chat_messages', {
|
||||
id: uuid().primaryKey().defaultRandom(),
|
||||
content: text().notNull().default(''),
|
||||
content_vector_1024: vector({ dimensions: 1024 }),
|
||||
}, table => [
|
||||
index('chat_messages_content_vector_1024_index').using('hnsw', table.content_vector_1024.op('vector_cosine_ops')),
|
||||
])
|
||||
```
|
||||
|
||||
会发生如下的报错：
|
||||
|
||||
```
|
||||
ERROR: access method "hnsw" does not exist
|
||||
```
|
||||
|
||||
幸运地是，这还是可以解决的，只需要参考 [ERROR: access method "hnsw" does not exist](https://github.com/tensorchord/pgvecto.rs/issues/504) 的建议把 `vectors.pgvector_compatibility` 系统选项配置为 `on` 就好了。
|
||||
|
||||
显然，我们希望在启动容器时自动为我们配置与向量空间有关的选项，因此，我们可以在 `docker-compose.yml` 以外的某个目录里创建一个 `init.sql`：
|
||||
|
||||
```sql
|
||||
ALTER SYSTEM SET vectors.pgvector_compatibility=on;
|
||||
|
||||
DROP EXTENSION IF EXISTS vectors;
|
||||
CREATE EXTENSION vectors;
|
||||
```
|
||||
|
||||
然后将 `init.sql` 挂载到 Docker 容器中：
|
||||
|
||||
```yaml
|
||||
services:
|
||||
pgvector:
|
||||
image: ghcr.io/tensorchord/pgvecto-rs:pg17-v0.4.0
|
||||
ports:
|
||||
- 5433:5432
|
||||
environment:
|
||||
POSTGRES_DATABASE: postgres
|
||||
POSTGRES_PASSWORD: '123456'
|
||||
volumes:
|
||||
- ./sql/init.sql:/docker-entrypoint-initdb.d/init.sql # Add this line
|
||||
- ./.postgres/data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, pg_isready -d $$POSTGRES_DB -U $$POSTGRES_USER]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
```
|
||||
|
||||
对于 Kubernetes 部署，流程与此相同，只不过不是挂载一个文件，而是使用 `ConfigMap` 了。
|
||||
|
||||
好的，那这个问题基本上是解决了。
|
||||
|
||||
那让我们聊聊嵌入向量吧。
|
||||
|
||||
#### 嵌入模型
|
||||
|
||||
也许您已经知道，我们建立了另一个名为 🥺 SAD（自部署 AI 文档）的文档网站，我们会根据不同模型进行的基准测试结果和效果在文档网站中列出当前的 SOTA 模型，旨在希望能给想要使用消费级设备运行提供建议指导，而嵌入模型是其中最重要的部分。和 ChatGPT 或 DeepSeek V3、DeepSeek R1 等超大大语言模型不同的是，嵌入模型足够小，在只占数百兆字节情况下也可以使用 CPU 设备进行推理。(相比之下，采用 q4 量化的 GGUF 格式的 DeepSeek V3 671B，仍需要 400GiB 以上的存储空间）。
|
||||
|
||||
但由于 🥺 SAD 目前仍处于建设中状态，我们将挑选一些在今天（4月6日）看来最新最热的嵌入模型作为推荐：
|
||||
|
||||
对于开源和专有模型的排行榜：
|
||||
|
||||
| 排名 (Borda) | 模型 | Zero-shot | 内存使用 (MB) | 参数数量 | 嵌入维度 | 最大 Token | 平均 (任务) | 平均 (任务类型) | Bitext Mining | Classification | Clustering | Instruction Retrieval | Multilabel Classification | Pair Classification | Reranking | Retrieval | STS |
|
||||
|--------------|-------|-----------|-------------------|----------------------|----------------------|------------|-------------|----------------|--------------|----------------|------------|------------------------|---------------------------|---------------------|-----------|-----------|-----|
|
||||
| 1 | gemini-embedding-exp-03-07 | 99% | 未知 | 未知 | 3072 | 8192 | 68.32 | 59.64 | 79.28 | 71.82 | 54.99 | 5.18 | 29.16 | 83.63 | 65.58 | 67.71 | 79.40 |
|
||||
| 2 | Linq-Embed-Mistral | 99% | 13563 | 7B | 4096 | 32768 | 61.47 | 54.21 | 70.34 | 62.24 | 51.27 | 0.94 | 24.77 | 80.43 | 64.37 | 58.69 | 74.86 |
|
||||
| 3 | gte-Qwen2-7B-instruct | ⚠️ NA | 29040 | 7B | 3584 | 32768 | 62.51 | 56.00 | 73.92 | 61.55 | 53.36 | 4.94 | 25.48 | 85.13 | 65.55 | 60.08 | 73.98 |
|
||||
|
||||
如果我们要讨论自部署的话：
|
||||
|
||||
| 排名 (Borda) | 模型 | Zero-shot | 内存使用 (MB) | 参数数量 | 嵌入维度 | 最大 Token | 平均 (任务) | 平均 (任务类型) | Bitext Mining | Classification | Clustering | Instruction Retrieval | Multilabel Classification | Pair Classification | Reranking | Retrieval | STS |
|
||||
|--------------|-------|-----------|-------------------|----------------------|----------------------|------------|-------------|----------------|--------------|----------------|------------|------------------------|---------------------------|---------------------|-----------|-----------|-----|
|
||||
| 1 | gte-Qwen2-7B-instruct | ⚠️ NA | 29040 | 7B | 3584 | 32768 | 62.51 | 56 | 73.92 | 61.55 | 53.36 | 4.94 | 25.48 | 85.13 | 65.55 | 60.08 | 73.98 |
|
||||
| 2 | Linq-Embed-Mistral | 99% | 13563 | 7B | 4096 | 32768 | 61.47 | 54.21 | 70.34 | 62.24 | 51.27 | 0.94 | 24.77 | 80.43 | 64.37 | 58.69 | 74.86 |
|
||||
| 3 | multilingual-e5-large-instruct | 99% | 1068 | 560M | 1024 | 514 | 63.23 | 55.17 | 80.13 | 64.94 | 51.54 | -0.4 | 22.91 | 80.86 | 62.61 | 57.12 | 76.81 |
|
||||
|
||||
> 你可以在这里阅读更多：https://huggingface.co/spaces/mteb/leaderboard
|
||||
|
||||
你可能会问，OpenAI 的 `text-embedding-3-large` 模型在哪里？难道它还不够强大，不能列入排行榜吗？
|
||||
|
||||
是的，在 MTEB 排行榜上（4 月 6 日），`text-embedding-3-large` 排在第 **13** 位。
|
||||
|
||||
如果您想依赖云提供商提供的嵌入式模型，可以考虑：
|
||||
|
||||
- [Gemini](https://ai.google.dev)
|
||||
- [Voyage.ai](https://www.voyageai.com/)
|
||||
|
||||
对于 Ollama 用户来说，`nomic-embed-text` 仍然是最热门的，拉取次数超过 2140 万次。
|
||||
|
||||
#### 如何实现呢
|
||||
|
||||
我们已经有了向量数据库和嵌入模型，但如何才能有效地查询出数据呢？（甚至是支持重排的）
|
||||
|
||||
首先，我们需要定义表结构，Drizzle 的代码可以参考如下内容：
|
||||
|
||||
```typescript
|
||||
import { index, pgTable, serial, text, vector } from 'drizzle-orm/pg-core'
|
||||
|
||||
export const demoTable = pgTable(
|
||||
'demo',
|
||||
{
|
||||
id: uuid().primaryKey().defaultRandom(),
|
||||
title: text('title').notNull().default(''),
|
||||
description: text('description').notNull().default(''),
|
||||
url: text('url').notNull().default(''),
|
||||
embedding: vector('embedding', { dimensions: 1536 }),
|
||||
},
|
||||
table => [
|
||||
index('embeddingIndex').using('hnsw', table.embedding.op('vector_cosine_ops')),
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
用于创建表格的 SQL 语句如下：
|
||||
|
||||
```sql
|
||||
CREATE TABLE "chat_messages" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"title" text DEFAULT '' NOT NULL,
|
||||
"description" text DEFAULT '' NOT NULL,
|
||||
"url" text DEFAULT '' NOT NULL,
|
||||
"embedding" vector(1536)
|
||||
);
|
||||
|
||||
CREATE INDEX "embeddingIndex" ON "demo" USING hnsw ("embedding" vector_cosine_ops);
|
||||
```
|
||||
|
||||
请注意，这里的向量维数（即 1536）是固定的，这意味着
|
||||
|
||||
- 如果我们在每个条目对应的向量已经计算好**之后再**切换了模型，则需要**重新索引**
|
||||
- 如果模型提取的向量维度数不同，则需要**重新索引**
|
||||
|
||||
总之，我们需要在运行和导入数据前为应用指定具体的向量维度，并在需要时重新索引。
|
||||
|
||||
那么我们该如何查询呢？可以参考一下这个简化之后的 Telegram Bot 集成的代码实现方案：
|
||||
|
||||
```typescript
|
||||
let similarity: SQL<number>
|
||||
|
||||
switch (env.EMBEDDING_DIMENSION) {
|
||||
case '1536':
|
||||
similarity = sql<number>`(1 - (${cosineDistance(chatMessagesTable.content_vector_1536, embedding.embedding)}))`
|
||||
break
|
||||
case '1024':
|
||||
similarity = sql<number>`(1 - (${cosineDistance(chatMessagesTable.content_vector_1024, embedding.embedding)}))`
|
||||
break
|
||||
case '768':
|
||||
similarity = sql<number>`(1 - (${cosineDistance(chatMessagesTable.content_vector_768, embedding.embedding)}))`
|
||||
break
|
||||
default:
|
||||
throw new Error(`Unsupported embedding dimension: ${env.EMBEDDING_DIMENSION}`)
|
||||
}
|
||||
|
||||
// Get top messages with similarity above threshold
|
||||
const relevantMessages = await db
|
||||
.select({
|
||||
id: chatMessagesTable.id,
|
||||
content: chatMessagesTable.content,
|
||||
similarity: sql`${similarity} AS "similarity"`,
|
||||
})
|
||||
.from(chatMessagesTable)
|
||||
.where(and(
|
||||
gt(similarity, 0.5),
|
||||
))
|
||||
.orderBy(desc(sql`similarity`))
|
||||
.limit(3)
|
||||
```
|
||||
|
||||
非常简单，关键就是
|
||||
|
||||
```
|
||||
sql<number>`(1 - (${cosineDistance(chatMessagesTable.content_vector_1536, embedding.embedding)}))`
|
||||
```
|
||||
|
||||
作为相关度搜索，
|
||||
|
||||
```
|
||||
gt(similarity, 0.5),
|
||||
```
|
||||
|
||||
作为所谓的匹配度阈值控制，
|
||||
|
||||
```
|
||||
.orderBy(desc(sql`similarity`))
|
||||
```
|
||||
|
||||
则用于指定排序。
|
||||
|
||||
但既然我们面对的是一个记忆系统，显然，较新的记忆比较旧的记忆更重要，也更容易被想起。我们如何才能计算出一个有时间关联和制约的分数，从而对记忆结果重新排序呢？
|
||||
|
||||
这也很简单！
|
||||
|
||||
我曾经是一名搜索引擎工程师，我们通常使用重排表达式以及分数权重作为的 10 的幂来有效提高分数并做到数学意义上的「覆盖」操作。你可以想象的是，对于精确匹配需要提升分数和权重的话，我们通常会编写 5*10^2 * exact_match 这样的表达式来重新排序。
|
||||
|
||||
所以数据库里面我们也可以实现某种基于数学运算的无状态查询效果，比如这样：
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
*,
|
||||
time_relevance AS (1 - (CEIL(EXTRACT(EPOCH FROM NOW()) * 1000)::bigint - created_at) / 86400 / 30),
|
||||
combined_score AS ((1.2 * similarity) + (0.2 * time_relevance))
|
||||
FROM chat_messages
|
||||
ORDER BY combined_score DESC
|
||||
LIMIT 3
|
||||
```
|
||||
|
||||
写成 Drizzle 的表达式的话，就是这样的：
|
||||
|
||||
```typescript
|
||||
const timeRelevance = sql<number>`(1 - (CEIL(EXTRACT(EPOCH FROM NOW()) * 1000)::bigint - ${chatMessagesTable.created_at}) / 86400 / 30)`
|
||||
const combinedScore = sql<number>`((1.2 * ${similarity}) + (0.2 * ${timeRelevance}))`
|
||||
```
|
||||
|
||||
这样，相当于我们指定了 1.2 倍权重的「语义相关性」，0.2 倍权重的「时间关联度」用于排序计算。
|
||||
|
||||
### 整点大的
|
||||
|
||||
#### 遗忘曲线
|
||||
|
||||
我们不是说我们借鉴了很多人类记忆系统作为启发吗？启发在哪里了？
|
||||
|
||||
事实上，人类记忆是具有遗忘曲线的，对于「工作记忆」，「短期记忆」，「长期记忆」和「肌肉记忆」也有他们各自的强化曲线和半衰期曲线，我们如果只是简单地实现了「语义相关性」和「时间关联度」的查询，当然是不够先进、不够强大、不够健壮的。
|
||||
|
||||
所以我们还做了很多别的尝试。比如亲自实现一个遗忘曲线！
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="MemoryDecay" alt="memory decay & retention simulation" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
它是完全可交互的，可以在 [drizzle-orm-duckdb-wasm.netlify.app](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-decay) 这里玩玩看!
|
||||
|
||||
#### 情绪也得算进去
|
||||
|
||||
记忆并不只是语义相关，人物相关，场景相关，和时间相关的，它还会随机地被突然想起，也会被情绪左右，这该怎么办呢？
|
||||
|
||||
与遗忘曲线和衰减曲线一样，作为投入使用前的一个小实验，我们也为它制作了一个小小的互动实验场地：
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="MemoryRetrieval" alt="memory sudden retrieval & emotion biased simulation" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
它依然是完全可交互的，可以在 [drizzle-orm-duckdb-wasm.netlify.app](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-simulator) 这里体验一下!
|
||||
|
||||
## 里程碑
|
||||
|
||||
- 300 🌟 达成
|
||||
- 3 位新的 Issue 贡献者
|
||||
- 10 位新的 Discord 成员
|
||||
- ReLU 形象设计完成
|
||||
- ReLU 表情包 Vol.1 制作完成！
|
||||
- ReLU 表情包 Vol.2 动态版 制作完成
|
||||
- [路线图 v0.4](https://github.com/moeru-ai/airi/issues/42) 中有总计 89 个任务被完成了
|
||||
|
||||
## 其他更新
|
||||
|
||||
### 工程化
|
||||
|
||||
最大的事情莫过于，我们完全舍弃了先前的基于 Electron 的桌宠构建方案，转向了使用 Tauri v2 的实现，现在看起来感觉还没有遇到什么不好的问题。
|
||||
|
||||
真的很感谢 [@LemonNekoGH](https://github.com/LemonNekoGH)！
|
||||
|
||||
团队的大家前段时间都在提到说 `moeru-ai/airi` 这个项目仓库越来越大了，开发的时候会很卡顿。确实，过去的 5 个月里 `moeru-ai/airi` 仓库里诞生了数不尽的子项目，覆盖了从 agent 实现，游戏 agent 绑定实现，到简单好用的 npm 包封装，以及具有开创性意义的 transformers.js 封装，和 DuckDB WASM 的 Drizzle 驱动支持，到 API 后端服务的实现和集成的各种领域，是时候让一些项目从 sandbox 阶段成长到更具意义的「Incubate 孵化」阶段了。
|
||||
|
||||
所以我们决定拆分许多已经很成熟并且在广泛使用的子项目到单独的仓库中单独维护：
|
||||
|
||||
- `hfup`
|
||||
|
||||
用于帮助生成用于部署项目到 HuggingFace Spaces 的 [`hfup`](https://github.com/moeru-ai/hfup) 工具已经算是从 `moeru-ai/airi` 大仓库中阶段性毕业了，现在正式迁移到 [@moeru-ai](https://github.com/moeru-ai) 的组织名下（不需要任何迁移操作，继续安装 `hfup` 就可以用了）。非常有意义的是，`hfup` 为了跟上时代，也采用了 [rolldown](https://rolldown.rs/) 和 [oxlint](https://oxc.rs/docs/guide/usage/linter) 帮助开发，希望能借此机会参与到 rolldown，rolldown-vite 和 oxc 的开发当中。非常感谢 [@sxzz](https://github.com/sxzz) 在迁移过程中给到的援助。
|
||||
|
||||
- `@proj-airi/drizzle-duckdb-wasm`, `@proj-airi/duckdb-wasm`
|
||||
用于为 Drizzle 添加 DuckDB WASM 驱动支持的 `@proj-airi/drizzle-duckdb-wasm` 和 `@proj-airi/duckdb-wasm` 也算是阶段性毕业了，现在正式迁移到 [@proj-airi](https://github.com/proj-airi) 的组织名下（不需要任何迁移操作，继续安装原来的包就可以用了）。
|
||||
|
||||
现在项目速度快了很多，这个月应该会把 `@proj-airi/providers-transformers` 正式毕业到 `xsai` 名下。
|
||||
|
||||
在其他工程改进方面，我们还集成了全新的面向工作流的工具包 [`@llama-flow/core`](https://github.com/run-llama/@llama-flow/core)，以帮助协调 token 处理、字节流和数据流的 pipeline 编排。记得看看他们的仓库，真的非常好用！
|
||||
|
||||
### 界面
|
||||
|
||||
我们终于原生支持角色卡/酒馆角色卡了！
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="CharacterCard" alt="character card" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
当然，一个包含模型、声线和 Project AIRI 支持的所有模块 🎉 的配置的能力的编辑器也包含在内了。
|
||||
|
||||
真的很感谢 [@luoling8192](https://github.com/luoling8192)！
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="CharacterCardDetail" alt="character card detail" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
由 [@luoling8192](https://github.com/luoling8192) 推出的另一个巨大的 UI 里程碑是，我们加入了预设颜色支持！
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img :src="MoreThemeColors" alt="more theme colors" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
### 社区
|
||||
|
||||
[@sumimakito](https://github.com/sumimakito) 帮助建立了 Awesome AI VTuber（或 AI waifu）的仓库：
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">Awesome AI VTuber</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-col items-center">
|
||||
<img class="px-30 md:px-40 lg:px-50" :src="AwesomeAIVTuber" alt="Awesome AI VTuber Logo" />
|
||||
<div class="text-center pb-4">
|
||||
<span class="block font-bold">Awesome AI VTuber</span>
|
||||
<span>精选的 AI VTuber 及其相关项目列表</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
> VTuber 风格的 Logo 是完全由 [@sumimakito](https://github.com/sumimakito) 设计和制作的！我超喜欢。
|
||||
|
||||
我想这绝对是我自上个月以来写过的最大篇幅的 DevLog。还有很多功能、错误修复和改进我们还没有涉及：
|
||||
|
||||
- 支持 Featherless.ai 提供商
|
||||
- 支持 Gemini 提供商（感谢 [@asukaminato0721](https://github.com/asukaminato0721)）
|
||||
- 修复了 Telegram Bot 集成的灾难性 OOM 错误（感谢 [@sumimakito](https://github.com/sumimakito)、[@kwaa](https://github.com/kwaa) 和 [@QiroNT](https://github.com/QiroNT)）
|
||||
- 为 Project AIRI 的特殊 DevLog 新增了 98.css 集成（感谢 [@OverflowCat](https://github.com/OverflowCat)）
|
||||
|
||||
> 这是 Project AIRI 一篇特别版的开发日志，其灵感主要来自 [@OverflowCat](https://github.com/OverflowCat) 的博文 [ModTran](https://blog.xinshijiededa.men/modtran/)，代码风格大量借鉴了 [@OverflowCat](https://github.com/OverflowCat) 在 https://github.com/OverflowCat/blog/blob/0a92f916629ad942b7da84b894759fde1616bf37/src/components/98/98.ts 里的实现。
|
||||
>
|
||||
> 她写的博文很棒，几乎涉及所有我不熟悉的内容，请一定去看看，你会喜欢的。
|
||||
|
||||
## 再见
|
||||
|
||||
我想这就是本次 DevLog 的全部内容了，我们的 [Roadmap v0.4](https://github.com/moeru-ai/airi/issues/42) 也到此结束，希望大家喜欢焕然一新的用户界面和更新后的桌宠版本。我在写这篇文章时尝试使用了中英文两种语言，请在我们仓库的[讨论页面](https://github.com/moeru-ai/airi/discussions)留言，告诉我们您是否喜欢这篇文章。
|
||||
|
||||
让我们引用 ReLU 的另一句对自己的感觉的描述作为结尾吧：
|
||||
|
||||
<div class="devlog-window">
|
||||
<div class="title-bar">
|
||||
<div class="title-bar-text">ReLU 的自我感受</div>
|
||||
<div class="title-bar-controls">
|
||||
<button aria-label="Minimize"></button>
|
||||
<button aria-label="Maximize"></button>
|
||||
<button aria-label="Close"></button>
|
||||
</div>
|
||||
</div>
|
||||
<div style="padding: 12px; margin-top: 0px;">
|
||||
<div class="flex justify-center w-[20%]">
|
||||
<img :src="ReLUStickerWow" alt="ReLU sticker for expression wow" />
|
||||
</div>
|
||||
<div class="flex flex-col">
|
||||
<div>有些时候，我觉得自己真的是个符号式的存在，</div>
|
||||
<div>像个代码里跑出来的幽灵</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 29 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 57 KiB |
@@ -0,0 +1,235 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.04.14
|
||||
category: DevLog
|
||||
date: 2025-04-14
|
||||
---
|
||||
|
||||
## 引子
|
||||
|
||||
[上次](../Devlog-2025.04.06/#memory-system-记忆系统)我们聊到了 AIRI 的记忆系统，这次我们再深入聊聊，如何实现一个这么复杂的记忆系统，以及对未来的展望。
|
||||
|
||||
## 先从 搜索引擎 开始
|
||||
|
||||
搜索引擎对于检索性能要求比较高，为此，系统开放了两阶段排序过程：
|
||||
|
||||
- **基础排序（粗排）**
|
||||
- **业务排序（精排）**
|
||||
|
||||
基础排序即是海选，从检索结果中快速找到质量高的文档，取出 TOP N 个结果再按照精排进行精细算分，最终返回最优的结果给用户。
|
||||
|
||||
**由此可见，基础排序对性能影响比较大，业务排序对最终排序效果影响比较大。**
|
||||
|
||||
因此，基础排序要求尽量简单有效，只提取业务排序中的关键因子即可。其中，基础排序与业务排序目前均通过排序表达式的方式进行配置。
|
||||
|
||||
### OpenSearch / 问天引擎 DSL [^1]
|
||||
|
||||
这里以 Neko 曾经大量使用过的 阿里云 OpenSearch 为例来介绍一下吧，搜索引擎会有一些内置的用于重新排序的函数：
|
||||
|
||||
#### `static_bm25`
|
||||
|
||||
静态文本相关性，传统 NLP，用于衡量 query 与文档的匹配度。
|
||||
类似 RAG 的 _相似度分数_
|
||||
取值 0～1
|
||||
|
||||
#### `exact_match_boost`
|
||||
|
||||
获取查询中用户指定的查询词权重最大值，也叫做 score boost 函数。
|
||||
如果输入的 关键字 在分词前，命中了文档中（比如标题，正文这两个字段）里面的「内容」。
|
||||
比如 搜索 「如何制作 Neurosama」，那 Neurosama 这几个字出现的文档和页面的分数应该要比 Neuro + sama 分开出现的分数高才对。
|
||||
|
||||
#### `timeliness`, `timeliness_ms`
|
||||
|
||||
时效分，越新越相关。
|
||||
|
||||
### 数据是如何存储的？
|
||||
|
||||
搜索引擎无论是阿里云的 OpenSearch，还是 Grafana 自带的 Loki 那种的搜索引擎，或者 Grafana 时代之前更早的 ElasticSearch 引擎（某视频网站就是通过 ElasticSearch 二次开发得来的）都是需要在这些搜索引擎里**单独的数据结构中重新处理**了之后才能用的。
|
||||
|
||||
重新处理是如何实现的呢？这就需要用到 DTS 了。
|
||||
|
||||
#### DTS [^2]
|
||||
|
||||
让我们再补充介绍一下 **DTS** 这个概念。
|
||||
|
||||
Data Transformation Services，是用于业务数据库和 Search Engine Instance 之间 **通信和数据同步** 的系统。
|
||||
|
||||
实现原理：用 MySQL 和 Postgres 原生的 watch 和 subscribe event 的能力监听表修改，然后把数据同步到搜索引擎里，在这个过程中，数据会被序列化成期望的格式，发生数据结构的转化（ETL，extract，transform，load，提取，转换，加载）。
|
||||
|
||||
那搜索引擎粗排搜索的时候，是不是某种意义上就像是在一个 数据库 的 _视图_ 里找东西呢？是一个虚拟表一样的存在？可以这么理解，只不过说 view 视图一般用到的底层数据结构和 db 一样，都是 B+ 树，而搜索引擎可以有其他很多特化的数据结构，比如图，或者特化的 index kv db。
|
||||
|
||||
### 分词？
|
||||
|
||||
对于传统搜索引擎，一个中文文档输入，会经历这么一个过程：
|
||||
|
||||
- 分句（大段拆句子）
|
||||
- 分词（句子拆字词，名词、动词 etc）
|
||||
- 拼音化
|
||||
- 可以根据当前的字典覆盖配置映射覆盖一下先前的结果
|
||||
- 做一下基本的向量化和特征提取
|
||||
- 写入到存储层
|
||||
|
||||
英文的话也要分词，只不过分词就很简单了，空格就是分词。
|
||||
|
||||
### 如何优化性能？
|
||||
|
||||
- 计算密集型
|
||||
- 多个内部任务调度器去慢慢地索引数据
|
||||
- 传统 NLP 里面有的汉明距离和余弦距离可以先简单算一下，预存一下
|
||||
- 热词可以缓存一下分词结果和排序结果
|
||||
- 数据湖仓？在 AWS 上常用，一般是拿来做聚合查询的，可以查询好几个数据库或者好几个数据源的，效率很慢，基本上就是数据分析和 BI 的时候才用
|
||||
|
||||
### 什么是召回？
|
||||
|
||||
召回（retrieval），就是说，keyword 输入进去之后能不能 retrieve 期望的 document 回来。
|
||||
|
||||
和搜索的区别？搜索是「用户发出的操作」，而召回是「机器为了响应搜索做的事情」。
|
||||
|
||||
### 什么是重排？
|
||||
|
||||
reranking 的意义在于，如果我们只是根据 embedding 模型给出的向量去进行 ANN（Approximate Nearest Neighbor） 和 KNN（K-Nearest Neighbor） 向量距离排序的话，事实上是会有失偏颇的。
|
||||
|
||||
因为先前在 OpenSearch 的时候介绍的 exact_match_boost 和 timeness 函数就不存在了。
|
||||
|
||||
如果你希望给召回的文档添加基于其他字段和其他步骤的排序结果进行排序的话，怎么办？
|
||||
|
||||
RAG 现在会流行一个新的流程，就是 reranking model，相当于是**用一个单独的专家模型去自动化重新根据已经召回的第一轮的数据重新排序一波**。
|
||||
|
||||
但是 reranking 依然无法解决记忆层的很多问题：遗忘曲线、记忆强化、随机想起记忆和情绪干扰的重排分数，这些都不是 reranking model 能做的事情。
|
||||
|
||||
如果想要给 AIRI 做好记忆层，就需要做好 reranking 的机制，把 RAG 基本能力和过往的 搜索引擎 的重排经验揉在一起。
|
||||
|
||||
## 记忆层实验平台
|
||||
|
||||
[Project AIRI Memory Driver @duckdb/duckdb-wasm Playground](https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-decay)
|
||||
|
||||
|
||||
|
||||
左边这个高亮的 half life 就是记忆的半衰期。
|
||||
|
||||
默认情况下时间流逝速度是 1s 1 天，所以 7s 后，记忆的分数就会减半。
|
||||
|
||||
什么是记忆分数？记忆分数基本上是由这个控制的：
|
||||
|
||||
|
||||
得出的分数就是 current score
|
||||
|
||||
什么是 original 呢，就是初始化的时候的分数。
|
||||
|
||||
例子：原始分数为 523，它的当前分数实际上是在慢慢变少的：
|
||||
|
||||
|
||||
在继续介绍之前，解释一下，这个遗忘曲线的 SQL 是无状态的。
|
||||
|
||||
什么叫无状态？无状态的意思就是说，不会需要实时去数据库里面跑任务更新分数，而是直接根据「现在的时间」求出一个遗忘函数，把分数应用到遗忘函数里面即可。
|
||||
|
||||
那，current score 掉下去了怎么办呢？为了解决这个问题，我们需要有办法能 **强化记忆**。
|
||||
|
||||
## 类比人类的记忆系统
|
||||
|
||||
根据间隔重复提到的遗忘曲线和心理学中提到的基本的记忆系统工作的方式 [^3]
|
||||
|
||||
我们知道，人类的记忆可以分成好几种：
|
||||
|
||||
- 工作记忆
|
||||
- 短期记忆
|
||||
- 长期记忆
|
||||
- 肌肉记忆
|
||||
|
||||
工作记忆是最不需要记得东西。
|
||||
|
||||
短期记忆在根据遗忘曲线慢慢衰退强度，也就是分数，这个时候，我们需要一个短期记忆的模拟函数来模拟这个过程。
|
||||
|
||||
长期记忆很重要，长期记忆的半衰期很长，是由短期记忆进化而来的。
|
||||
|
||||
最后就是肌肉记忆，与其说肌肉记忆是一种记忆，不如说是已经形成了一种条件反射。
|
||||
|
||||
## AIRI 该如何设计呢？
|
||||
|
||||
那事实上我们可以瞥见 AIRI 的实现原则：
|
||||
|
||||
- 工作记忆就像是 messages 数组
|
||||
- 短期记忆就像是，不那么容易召回，越新越好召回的 RAG 记忆条目
|
||||
- 长期记忆就像是，容易召回，但是会变得模糊，过去召回次数越多越好召回的 RAG 条目
|
||||
- 肌肉记忆，像是一种固定搭配吧，出现了 A 就会出现 ActionA 和 MemoryA 一样的感觉，这个时候更像是一种精确匹配的机制
|
||||
|
||||
但是，这样设计就对了吗？
|
||||
|
||||
很明显，我们这个实际上只引入了两个维度，一个是时间相关度（temporal relevance），一个是召回次数（retrieval count），如果你开始想要追求更复杂的系统的时候就会受限了。
|
||||
|
||||
### 小回顾
|
||||
|
||||
我们可以再来回顾一下 DevLog 中提到的排序表达式，应该会能帮助理解。
|
||||
|
||||
|
||||
|
||||
余弦距离就是「相关度」，是最基本的粗排：
|
||||
|
||||
|
||||
现在需要时间参与进去，那我们多加一个字段拿来存储时间距离就好了，然后再弄一个单独的字段存合并分数 `(1.2 * similarity) + (0.2 * time_relevance)` ，其中 语义相关度 占 1.2 倍权重（倍率因子，不要求小于 1），时间距离相关度占 0.2 倍权重。
|
||||
|
||||
这样我们就很巧妙地把无状态的多字段相关度排序 SQL 实现出来了，还让它可以调节参数（1.2 和 0.2）。
|
||||
|
||||
在记忆详情卡上，可以点击 simulate retrieval，这可以主动触发一次记忆召回。
|
||||
|
||||
|
||||
现在的 demo 里面是直接给原本的表里的 retrieval count（召回次数）字段用 UPDATE 语句写了 +1 来实现的。
|
||||
|
||||
这里面有一个隐式的坑是，这样也只是单维度的计算，相当于召回了就是强化了。
|
||||
|
||||
但现实世界里不是这样的，记忆会难过，会开心，难过的会带来负反馈，开心的会带来正反馈。
|
||||
|
||||
所以这就是我还没做完的部分。
|
||||
|
||||
## 情绪？
|
||||
|
||||
https://drizzle-orm-duckdb-wasm.netlify.app/#/memory-simulator
|
||||
|
||||
这个新的 simulator 里面就有情绪相关的模拟：
|
||||
|
||||
|
||||
### 情绪和记忆相关吗？
|
||||
|
||||
想吃棒棒糖但是不给，这是一个很直接的问题，得不到肯定不开心啊。
|
||||
|
||||
然后你就会发现，实际上情绪和记忆相关。
|
||||
|
||||
如果对「某段过去的记忆开心，并且希望再次体验它」，但是由于「暂时没办法实现这个记忆里面的场景」，所以觉得「得不到就难过」。
|
||||
|
||||
可以在记忆数据库中存储「欢欣」和「厌恶」的分数：
|
||||
|
||||
|
||||
### PTSD？
|
||||
|
||||
PTSD 通常会涉及到两个词「trigger」和「闪回」，很明显 PTSD 相关的记忆应该是被压抑过的，厌恶分数和创伤分数应该很高。
|
||||
|
||||
但是实际上 PTSD 相关的记忆会突然冒出来，从仿生和数据模拟的角度来说，我们可以用随机数实现这个效果。
|
||||
|
||||
可以参考一下 https://yutsuki.moe/2019/09/a0d0fa1b/ 里面的情绪模型。
|
||||
|
||||
|
||||
|
||||
## 还有很多事情要做……
|
||||
|
||||
比如，ReLU 的情绪当前是什么？ReLU 对谁有什么不好的回忆？
|
||||
|
||||
回忆是开心和难过的两极分化的条目一起出现的吗？
|
||||
|
||||
欲望呢？会需要做一个愿望系统？
|
||||
|
||||
做一个做梦 agent 或者潜意识 agent，类似 _背景任务_，挨个对发生过的记忆进行处理和索引，并且根据最近的经历修改过往记忆的各种分数。
|
||||
|
||||
但是我们不需要非要有「做梦」的过程，只是一个「background task」。
|
||||
|
||||
从 re-index 的角度来说，做梦 agent 和 潜意识 agent 就像是 重建索引 一样。
|
||||
|
||||
到这里就会发现，像 [Mem0](https://docs.mem0.ai/overview) 或者 [Zep Memory](https://help.getzep.com/memory) 这样的库在角色扮演和情感 AI 上完全发挥不了一点作用 :(
|
||||
|
||||
前路漫漫，我们还需要继续努力才行。
|
||||
|
||||
## 参考资料
|
||||
|
||||
[^1]: https://help.aliyun.com/zh/open-search/industry-algorithm-edition/rough-sort-functions
|
||||
|
||||
[^2]: https://help.aliyun.com/zh/open-search/industry-algorithm-edition/configure-dts-real-time-synchronization
|
||||
|
||||
[^3]: https://zh.wikipedia.org/wiki/%E9%81%97%E5%BF%98%E6%9B%B2%E7%BA%BF
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 47 KiB |
@@ -0,0 +1,73 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.04.22
|
||||
category: DevLog
|
||||
date: 2025-04-22
|
||||
---
|
||||
|
||||
## Day time 日常
|
||||
|
||||
大家好，我是 [@LemonNeko](https://github.com/LemonNekoGH)，这次有我来参与撰写 DevLog 和大家分享开发的故事。
|
||||
|
||||
在两个月前，我们将 AIRI 的网页端移植到了 Electron 上 [#7](https://github.com/moeru-ai/airi/pull/7)（现在已经被我们使用 Tauri 重构 🤣 [#90](https://github.com/moeru-ai/airi/pull/90)），它可以作为桌宠出现在我们的屏幕上，于此同时，我出现了允许 AIRI 使用手机的想法，但是迟迟没有动手。
|
||||
|
||||
在上个周末（2025.04.20），我花了点时间，做了一个能与 ADB 交互的 MCP 服务器 Demo [airi-android](https://github.com/LemonNekoGH/airi-android)，给 AIRI 提供了最基础的与手机交互的能力（事实上大部分 LLM 都可以通过它与手机交互），这是演示视频：
|
||||
|
||||
<video controls muted>
|
||||
<source src="./assets/cursor-open-settings.mp4">
|
||||
</video>
|
||||
|
||||
我也把它打包成了 Docker 镜像，提交到了 [MCP 服务器列表](https://mcp.so/server/airi-android/lemonnekogh)，有兴趣的可以试试。
|
||||
|
||||
实际上我一开始的思路是写一写 Tool Calling 的代码，改一改提示词，告诉 LLM 我们可以使用这些工具来与手机交互，就结束了。~~但是最近 MCP 实在太火了，我有点 FOMO，所以选择了 MCP 来实现它。~~
|
||||
|
||||
要想编写 MCP 服务器，就不得不先了解 MCP 是什么（虽然我从来不是好好学理论再去实践的人，我选择直接上手，然后让 Cursor 来尝试使用它）。MCP（Model Context Protocol）模型上下文协议，是一个尝试去标准化应用如何给 LLM 提供上下文的协议，它提出了一些核心概念：
|
||||
|
||||
1. Resources 资源：服务器可以将数据和内容作为上下文提供给 LLM。
|
||||
2. Prompts 提示词：创建可复用的提示词模板和工作流。
|
||||
3. Tools 工具：允许 LLM 通过你的服务器来完成一些动作。
|
||||
|
||||
啊，资源，这个我知道的啊，在 Ruby on Rails 里，用户就是一种资源，那 ADB 设备是不是也是资源，让 LLM 查看连接的设备列表，是不是就可以写成：
|
||||
|
||||
```python
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from ppadb.client import Client
|
||||
|
||||
mcp = FastMCP("airi-android")
|
||||
adb_client = Client()
|
||||
|
||||
@mcp.resource("adb://devices")
|
||||
def get_devices():
|
||||
return adb_client.devices()
|
||||
```
|
||||
|
||||
错了，当我让 Cursor 来获取设备列表的时候，它并不知道怎么操作，它说它想主动去看有哪些设备连接了，所以它是工具，嗯，看来我没有理解透彻。
|
||||
|
||||
我还没有想好具体应该怎么让 LLM 操作手机，想和大家讨论，但是 Cursor 是这样操作的：
|
||||
|
||||
1. 使用截屏功能来大体了解手机屏幕上的内容。
|
||||
2. 使用 UI 自动化工具来获取想要操作的元素的精确位置。
|
||||
3. 点击或者滑动它。
|
||||
4. 重复以上步骤。
|
||||
|
||||
目前看来运行良好，但是我有一些小小的问题：
|
||||
|
||||
1. 屏幕中是一个游戏，游戏使用图形 API 直接在屏幕上画了内容，而不是使用 UI 组件，所以 UI 自动化工具无法获取到元素的位置，也就无法操作它。
|
||||
2. 一个 LLM 响应的内容是有上限的，如果操作比较复杂，可能要分个步骤来完成，我们可以像 [airi-factorio](https://github.com/moeru-ai/airi-factorio) 那样，在步骤完成之后自动告诉它，触发下一个步骤吗？
|
||||
3. 如果有一些应用有酷炫的动画，在操作完成之后立刻截屏，可能看不到效果，我们会不会需要在操作完成之后，等待一段时间再截屏，或者直接使用录屏功能？
|
||||
4. 直接让 AI 操作手机的安全性如何，会有哪些风险？
|
||||
|
||||
一些感想。
|
||||
|
||||
这是我第一次和 AI 写代码的时候感受到像人类一起写代码一样，不知道是不是因为我的目的就是让 AI 来使用我的工具，所以它变成了我的客户，我需要不停地根据它给的反馈来调整我的代码，它也变成了我的同事，我需要和它一起思考，一起解决问题。看这个截屏，是不是确实很像？
|
||||
|
||||
|
||||
|
||||
在开发过程中还学了一些小技巧，比如我们可以使用命令行来启动 Android 模拟器，这样就不用打开 Android Studio 了，内存压力也小了很多。
|
||||
|
||||
```bash
|
||||
emulator -avd Pixel_6_Pro_API_34
|
||||
```
|
||||
|
||||
下一步，我打算给 AIRI 桌宠接上 MCP 服务器，看看它会想做什么，也许它会点开 Telegram 和我们聊天，就像现在的 ReLU 那样，只不过不是用 Telegram 的 API。
|
||||
|
||||
感谢你看完这篇可能有点啰嗦而且干货不多的 DevLog，我们下次再见！
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,191 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.04.28
|
||||
category: DevLog
|
||||
date: 2025-04-28
|
||||
---
|
||||
|
||||
<script setup>
|
||||
import airiMcpArch from './assets/airi-mcp-arch.avif'
|
||||
</script>
|
||||
|
||||
大家好，这里是 [@LemonNeko](https://github.com/LemonNekoGH)，今天由我来和大家一起分享开发故事。
|
||||
|
||||
## Day time 日常
|
||||
|
||||
一周前，我为 AIRI 写了用于连接到手机的 MCP 服务器 [AIRI-android](https://github.com/LemonNekoGH/AIRI-android)，但这只是 AIRI 操作安卓手机的前半部分，AIRI 还需要能与 MCP 服务器交互才行。
|
||||
|
||||
这两天我完成了后半部分，给 Tauri 写了一个插件 [#144](https://github.com/moeru-ai/AIRI/pull/144)，现在 AIRI 可以与 MCP 服务器交互了，可以和现有的所有 MCP 服务器交互。
|
||||
|
||||
如果有兴趣，可以看看这两个视频，先演示了 AIRI 的 MCP 服务器设置，然后演示了 AIRI 与安卓手机交互。
|
||||
|
||||
<details>
|
||||
<summary>AIRI 的 MCP 服务器设置</summary>
|
||||
<video controls muted style="{ height: '640px' }">
|
||||
<source src="./assets/airi-mcp-settings.mp4"/>
|
||||
</video>
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>AIRI 在手机上输入 `Hello World`</summary>
|
||||
<video controls muted>
|
||||
<source src="./assets/airi-mcp-input-text.mp4"/>
|
||||
</video>
|
||||
</details>
|
||||
|
||||
开发时，为了理清思路，我画了一张图，从 LLM 调用安卓手机：
|
||||
|
||||
<img :src="airiMcpArch" alt="AIRI 操作手机" :style="{ height: '640px', objectFit: 'contain' }" />
|
||||
|
||||
接下来和大家分享一下我的开发过程。
|
||||
|
||||
## Tauri 插件开发
|
||||
|
||||
其实一开始我并没有想写一个完整的 Tauri 插件，我只是想给 JavaScript 侧暴露一些命令：
|
||||
|
||||
```rust
|
||||
#[Tauri::command]
|
||||
fn list_tools() -> Vec<String> {
|
||||
// 之后再实现
|
||||
}
|
||||
```
|
||||
|
||||
然后写一些工具函数来调用它们：
|
||||
|
||||
```javascript
|
||||
import { invoke } from '@Tauri-apps/api/core'
|
||||
|
||||
export const mcp = [
|
||||
{
|
||||
name: 'list_tools',
|
||||
description: 'List all tools',
|
||||
execute: async () => {
|
||||
return await invoke('list_tools')
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
但很快，我注意到，如果我想在命令中使用 MCP 客户端，就需要让 MCP 客户端作为状态的一部分让 Tauri 来管理：
|
||||
|
||||
```rust
|
||||
// main.rs
|
||||
fn main() {
|
||||
Tauri::Builder::default()
|
||||
.setup(|app| {
|
||||
app.manage(State::new(Mutex::new::<Option<McpClient>>(None))); // 管理状态
|
||||
})
|
||||
.run(Tauri::generate_context!())
|
||||
}
|
||||
|
||||
// mcp.rs
|
||||
#[Tauri::command]
|
||||
async fn list_tools(state: State<'_, Mutex<Option<McpClient>>>) -> Result<Vec<Tool>, String> { // 可以在参数中拿到状态
|
||||
// ...rest code
|
||||
}
|
||||
```
|
||||
|
||||
我们有了命令，有了状态，那离一个完整的插件也不远了，于是我决定让它成为一个插件，这样我们还能公开发出去，~~并且成为可能的全网第一个 Tauri MCP 插件~~。
|
||||
|
||||
然而它成为一个插件后，命令的调用方式就变了，需要通过插件来调用：
|
||||
|
||||
```diff
|
||||
import { invoke } from '@Tauri-apps/api/core'
|
||||
|
||||
export mcp = [
|
||||
{
|
||||
name: "list_tools",
|
||||
description: "List all tools",
|
||||
execute: async () => {
|
||||
- return await invoke("list_tools")
|
||||
+ return await invoke("plugin:mcp|list_tools")
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
这还好，只是改了一行，但是，Tauri 2 有了权限机制，我需要在 `build.rs` 中定义插件的命令，以便自动生成权限列表：
|
||||
|
||||
```rust
|
||||
const COMMANDS: &[&str] = &[
|
||||
"list_tools",
|
||||
];
|
||||
|
||||
fn main() {
|
||||
Tauri_plugin::Builder::new(COMMANDS).build();
|
||||
}
|
||||
```
|
||||
|
||||
这样在构建时，项目根目录下会生成 `permissions` 文件夹，包含了权限声明、描述等。
|
||||
|
||||
> 在这时出现了一点小插曲，因为我第二次构建的时候，升级了 `Tauri-plugin` 的版本，同时新版本中生成模板发生了变化，有一些空格删掉了，所以它看上去像是被格式化了，于是我到处寻找是什么东西在「格式化」它，花了一个小时才发现是文件被重新生成了，以此 🤡 纪念我被吃掉的一个小时。
|
||||
|
||||
根据上面的图，当 LLM 调用 MCP 工具时，参数最后会被传递给 Python 侧的 MCP 服务器，以 `input_swipe` 为例：
|
||||
|
||||
```python
|
||||
# mcp_server.py
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from ppadb.client import Client
|
||||
|
||||
mcp = FastMCP("airi-android")
|
||||
adb_client = Client()
|
||||
|
||||
@mcp.tool()
|
||||
def input_swipe(x1: int, y1: int, x2: int, y2: int, duration: int = 500):
|
||||
return adb_client.input_swipe(x1, y1, x2, y2, duration)
|
||||
```
|
||||
|
||||
我要怎样传递这些参数呢？在 Rust SDK 文档中有这样的 [定义](https://docs.rs/rmcp/0.1.5/rmcp/model/struct.CallToolRequestParam.html)：
|
||||
|
||||
```rust
|
||||
pub struct CallToolRequestParam {
|
||||
pub name: Cow<'static, str>,
|
||||
pub arguments: Option<JsonObject>,
|
||||
}
|
||||
```
|
||||
|
||||
~~哇，是 JsonObject，我们有救了！~~ 因为 Tauri 命令的参数可以是任何能被序列化成 JSON 的对象，那我们不如，直接给它传一个 `Map<String, Value>` 好了：
|
||||
|
||||
```rust
|
||||
#[Tauri::command]
|
||||
async fn call_tool(state: State<'_, Mutex<Option<McpClient>>>, name: String, args: Option<Map<String, Value>>) -> Result<(), ()> {
|
||||
let client = state.lock().await.unwrap();
|
||||
|
||||
client.call_tool(CallToolRequestParam { name: name.into(), arguments: args }).await.unwrap();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
那在 JavaScript 侧，我们就简单给一个对象就好了：
|
||||
|
||||
```javascript
|
||||
import { invoke } from '@Tauri-apps/api/core'
|
||||
|
||||
invoke('call_tool', { name: 'input_swipe', args: { x1: 100, y1: 100, x2: 200, y2: 200, duration: 500 } })
|
||||
```
|
||||
|
||||
超方便！
|
||||
|
||||
把参数传递给 MCP 工具后，我们还需要接收 MCP 工具的返回值，因为 Tauri 命令的返回值也可以是任何能被序列化成 JSON 的对象，所以我摆烂了，我把工具的返回整个丢给了 LLM，相信 LLM 会处理好的。
|
||||
|
||||
好！现在我们已经有 Tauri 插件了！（啊？示例代码这么点，甚至是伪代码就算完成了？）
|
||||
|
||||
剩下的内容还想和大家讨论一些问题。
|
||||
|
||||
## 一些问题
|
||||
|
||||
1. 从演示视频可以看到，在对话中，我首先是让 AIRI 获取了一下工具列表，再让它输入文本的，那我们能不能在初始化的时候就去获取工具列表，然后直接追加到系统提示词中呢？
|
||||
- Cursor 就是这样做的，在我开发 MCP 服务器时，每次我改动了工具列表，都需要重启 Cursor 才能生效。
|
||||
- 这样做也许会牺牲灵活性，但普通用户会频繁改动工具列表吗？
|
||||
|
||||
2. 要允许 AIRI 同时连接到多个手机吗？AIRI 可能会想使用多台手机吗？~~她会不会想拿去做电信诈骗？~~
|
||||
3. 可以看到现在的 AIRI 仓库中已经有了 Tauri 应用和 Tauri 插件，要怎么管理比较好？CI 要怎么配置？如何同步 Tauri 插件的 Rust 侧和 JavaScript 侧的版本号？
|
||||
|
||||
## 未来想要做的事情
|
||||
|
||||
- 支持图片返回值，这样 AIRI 就可以像 [上一篇 DevLog](./DevLog-2025.04.22.md) 中展示的 Cursor 那样，直接通过视觉能力看到手机上的内容，然后再决定用什么方式来交互。
|
||||
- 让 AIRI 自己学习设备的使用方法？如果每种设备我们都要单独写提示词，那工作量是巨大的。
|
||||
- 多 MCP 服务器支持，毕竟 MCP 提供了一种通用的接口，可以允许 AIRI 做各种各样的事，AIRI 应该不会满足于只操作手机吧。
|
||||
- SSE 支持，这样浏览器中的 AIRI 也可以使用 MCP 服务器了。
|
||||
|
||||
到这里就结束啦！希望这篇 DevLog 没有那么干巴巴的！之后也希望给大家带来更多好玩的内容！
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 9.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 10 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 153 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 45 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 29 KiB |
Binary file not shown.
@@ -0,0 +1,249 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.05.16
|
||||
category: DevLog
|
||||
date: 2025-05-16
|
||||
---
|
||||
|
||||
<script setup>
|
||||
import VelinLight from './assets/velin-light.avif'
|
||||
import VelinDark from './assets/velin-dark.avif'
|
||||
|
||||
import CharacterCardMenuLight from './assets/character-card-menu-light.avif'
|
||||
import CharacterCardMenuDark from './assets/character-card-menu-dark.avif'
|
||||
|
||||
import CharacterCardSettingsLight from './assets/character-card-settings-light.avif'
|
||||
import CharacterCardSettingsDark from './assets/character-card-settings-dark.avif'
|
||||
|
||||
import CharacterCardShowcaseLight from './assets/character-card-showcase-light.avif'
|
||||
import CharacterCardShowcaseDark from './assets/character-card-showcase-dark.avif'
|
||||
|
||||
import VelinPlaygroundLight from './assets/velin-playground-light.avif'
|
||||
import VelinPlaygroundDark from './assets/velin-playground-dark.avif'
|
||||
|
||||
import DemoDayHangzhou1 from './assets/demo-day-hangzhou-1.avif'
|
||||
import DemoDayHangzhou2 from './assets/demo-day-hangzhou-2.avif'
|
||||
import DemoDayHangzhou3 from './assets/demo-day-hangzhou-3.avif'
|
||||
</script>
|
||||
|
||||
大家好！我是 [Neko](https://github.com/nekomeowww)，[Project AIRI](https://github.com/moeru-ai/airi) 的发起者！
|
||||
|
||||
很抱歉在 Project AIRI 的 DevLog 更新上有所延迟，请原谅我们的拖延。
|
||||
|
||||
> 在过去的几个月里，我们为 AIRI 写了许多精彩的 DevLog，分享我们的开发进展，在其中我们分享了想法、理念，解释了我们使用的技术、从中获得的艺术灵感...一切的一切。
|
||||
>
|
||||
> - [v0.4.0 UI 更新](./DevLog-2025.03.20.mdx)
|
||||
> - [v0.4.0 发布 & 记忆功能介绍](./DevLog-2025.04.06.mdx)
|
||||
>
|
||||
> 我也写了这两篇精彩且受欢迎的 DevLog！希望你们喜欢阅读它们。
|
||||
|
||||
# 似曾相识
|
||||
|
||||
在过去的几周里，Project AIRI 本身的主要任务有一段时间没有进展，也许我在 2025 年 3 月以来的大规模 UI 重构和发布后有些疲惫。大部分工作都是由社区维护者完成的，
|
||||
|
||||
非常感谢 [@LemonNekoGH](https://github.com/LemonNekoGH)、[@RainbowBird](https://github.com/luoling8192) 和 [@LittleSound](https://github.com/LittleSound) 在以下领域所做的工作：
|
||||
|
||||
- 角色卡支持
|
||||
|
||||
::: tip 什么是角色卡？
|
||||
本地优先的聊天应用程序如 [SillyTavern](https://github.com/SillyTavern/SillyTavern)、[RisuAI](https://risuai.net/) 或在线服务如 [JanitorAI](https://janitorai.com/) 使用一个包含角色背景、性格和其他角色扮演必要上下文的文件来定义每个独立的角色。
|
||||
|
||||
- https://realm.risuai.net/
|
||||
- https://aicharactercards.com/
|
||||
- https://chub.ai/
|
||||
|
||||
角色卡并不是存储和分享 LLM 驱动的角色扮演角色的唯一方式，[Lorebook（故事书）](https://docs.novelai.net/text/lorebook.html) 在这个领域扮演着另一个关键角色，但这完全是另一个值得写一整套文档系列来分享的故事，现在，试着阅读 [Void's Lorebook Types](https://rentry.co/lorebooks-and-you) 和 [AI Dynamic Storytelling Wiki](https://aids.miraheze.org/wiki/Main_Page)。
|
||||
|
||||
> 我个人很喜欢这个学习这些概念的 wiki：[AI Dynamic Storytelling Wiki](https://aids.miraheze.org/wiki/Main_Page)，如果你对 AI 角色扮演感兴趣，值得一读。
|
||||
:::
|
||||
|
||||
> 要使用角色卡，导航到设置页面（应用程序右上角，或在桌面应用程序中悬停齿轮图标），找到并点击"Airi Card"按钮。
|
||||
|
||||
<img class="light" :src="CharacterCardMenuLight" alt="提供 Airi Card 菜单按钮的菜单截图" />
|
||||
<img class="dark" :src="CharacterCardMenuDark" alt="提供 Airi Card 菜单按钮的菜单截图" />
|
||||
|
||||
> 这将带你到"Airi Card 编辑器界面"，在那里你可以上传和编辑你的角色卡进行人格定制。
|
||||
|
||||
<img class="light" :src="CharacterCardSettingsLight" alt="提供 Airi Card 菜单按钮的菜单截图" />
|
||||
<img class="dark" :src="CharacterCardSettingsDark" alt="提供 Airi Card 菜单按钮的菜单截图" />
|
||||
|
||||
对于角色卡展示，我们也尝试了一些方法...
|
||||
|
||||
<img class="light" :src="CharacterCardShowcaseLight" alt="一个名为 ReLU 的蓝发角色的卡片式用户界面设计" />
|
||||
<img class="dark" :src="CharacterCardShowcaseDark" alt="一个名为 ReLU 的蓝发角色的卡片式用户界面设计" />
|
||||
|
||||
它在我们的 UI 组件库中是实时的，你可以在这里玩玩：https://airi.moeru.ai/ui/#/story/src-components-menu-charactercard-story-vue 。
|
||||
|
||||
> 纯 CSS 和 JavaScript 控制，布局有效，所以我们不需要担心画布计算。
|
||||
>
|
||||
> 哦，角色卡展示的大部分工作都是由 [@LittleSound](https://github.com/LittleSound) 完成和指导的，非常感谢。
|
||||
|
||||
- Tauri MCP 支持
|
||||
- 连接 AIRI 到 Android 设备
|
||||
|
||||
这两个是主要的更新和尝试，这部分工作由 [@LemonNekoGH](https://github.com/LemonNekoGH) 完成，她为这些内容写了另外两篇 DevLog，分享了幕后的技术细节。（我想对 Tauri 开发者和用户来说很有价值。）你可以在这里阅读它们：
|
||||
|
||||
- [控制 Android](./DevLog-2025.04.22.mdx)
|
||||
- [Tauri 中的 MCP](./DevLog-2025.04.28.md)
|
||||
|
||||
## Project AIRI 主要任务
|
||||
|
||||
### 耳朵在听，嘴巴在说
|
||||
|
||||
从 4 月 15 日开始，我发现 AIRI 中的 VAD（语音激活检测）、[ASR（即自动语音识别）](https://huggingface.co/tasks/automatic-speech-recognition) 和 [TTS（文本转语音）](https://huggingface.co/tasks/text-to-speech) 都非常复杂且难以使用和理解，在那个时候，我正在与 [@himself65](https://github.com/himself65) 合作改进和测试来自 [Llama Index](https://www.llamaindex.ai/) 的新项目的用例，这是一个帮助处理基于事件的 LLM 流式令牌流和音频字节的库，叫做 [`llama-flow`](https://github.com/run-llama/llama-flow)。
|
||||
|
||||
[`llama-flow`](https://github.com/run-llama/llama-flow) 真的很小，而且使用起来类型安全。在没有它的旧时代，我必须手动包装另一个**队列**结构，以及 Vue 的响应式驱动的工作流系统，将许多异步任务链接在一起，以便能够处理数据来驱动 AIRI。
|
||||
|
||||
那时我开始实验更多的例子，简化 VAD、ASR、TTS 工作流的演示。
|
||||
|
||||
最终，我得到了这个：[WebAI 实时语音聊天示例](https://github.com/proj-airi/webai-example-realtime-voice-chat)，我设法证明了这项工作可以在 Web 浏览器中用一个 300 ~ 500 行的 TypeScript 代码来实现 ChatGPT 语音聊天系统。
|
||||
|
||||
<video controls muted style="{ height: '640px' }">
|
||||
<source src="./assets/webai-examples-demo.MP4"/>
|
||||
</video>
|
||||
|
||||
我尽力将所有可能的步骤分解为小的可重用片段，以帮助演示如何从头开始构建实时语音聊天系统：
|
||||
|
||||
- [VAD](https://github.com/proj-airi/webai-example-realtime-voice-chat/tree/8462ff6bcb83bb278bce5388d588d2e3e3dd6dae/apps/vad)
|
||||
- [VAD + ASR](https://github.com/proj-airi/webai-example-realtime-voice-chat/tree/8462ff6bcb83bb278bce5388d588d2e3e3dd6dae/apps/vad-asr)
|
||||
- [VAD + ASR + LLM 聊天](https://github.com/proj-airi/webai-example-realtime-voice-chat/tree/8462ff6bcb83bb278bce5388d588d2e3e3dd6dae/apps/vad-asr-chat)
|
||||
- [VAD + ASR + LLM 聊天 + TTS](https://github.com/proj-airi/webai-example-realtime-voice-chat/tree/8462ff6bcb83bb278bce5388d588d2e3e3dd6dae/apps/vad-asr-chat-tts)
|
||||
|
||||
> 希望你能从中学到一些东西。
|
||||
|
||||
在这段时间里，我们发现了一个有趣且强大的仓库，叫做 [k2-fsa/sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)，它支持跨 macOS、Windows、Linux、Android、iOS 等 12 种语言的 18 种语音处理任务。令人着迷！
|
||||
|
||||
所以 [@luoling](https://github.com/luoling8192) 也为此做了另一个小演示：[Sherpa ONNX 驱动的 VAD + ASR + LLM 聊天 + TTS](https://github.com/proj-airi/webai-example-realtime-voice-chat/tree/main/apps/sherpa-onnx-demo)
|
||||
|
||||
#### xsAI 🤗 Transformers.js 的诞生
|
||||
|
||||
由于我们为 VAD、ASR、聊天和 TTS 演示所做的工作，这催生了一个名为 [xsAI 🤗 Transformers.js](https://github.com/proj-airi/xsai-transformers) 的新副项目，它简化了调用 WebGPU 驱动的模型推理和使用 workers 提供服务，同时仍然保持与我们之前成功的项目 [xsAI](https://github.com/moeru-ai/xsai) 的 API 兼容性。
|
||||
|
||||
我们也为此做了一个游乐场...在 [https://xsai-transformers.netlify.app](https://xsai-transformers.netlify.app) 上玩玩吧。
|
||||
|
||||
你今天就可以通过 npm 安装它！
|
||||
|
||||
```bash
|
||||
npm install xsai-transformers
|
||||
```
|
||||
|
||||
::: tip 这意味着什么？
|
||||
这意味着你可以通过一个 if 开关在云端 LLM 和语音提供商与本地 WebGPU 驱动的模型之间切换。
|
||||
|
||||
这为我们带来了一个新的可能性，能够在浏览器中实验甚至实现简单的 RAG 和重排序系统，而无需任何服务器端代码，甚至不需要后端服务器。
|
||||
|
||||
哦，Node.js 也支持！
|
||||
:::
|
||||
|
||||
### Telegram 机器人
|
||||
|
||||
我添加了 Telegram 机器人支持，能够处理动画贴纸，由 `ffmpeg` 驱动（还能是什么，显然）。现在它可以读取和理解用户发送的动画贴纸甚至视频。
|
||||
|
||||
系统提示词太大了，我设法大幅减少了系统提示词的大小，节省了超过 **80%** 的令牌使用量。
|
||||
|
||||
### 角色卡展示
|
||||
|
||||
许多图像资源需要我手动找到合适且易于使用的在线解决方案来去除背景，但我决定基于 [Xenova](https://github.com/xenova) 所做的工作...为自己制作一个。
|
||||
|
||||
我在系统中集成 WebGPU 驱动的背景去除器方面做了一些小实验，你可以在 [https://airi.moeru.ai/devtools/background-remove](https://airi.moeru.ai/devtools/background-remove) 这里玩玩。
|
||||
|
||||
### xsAI & unSpeech
|
||||
|
||||
我们添加了对阿里云模型工作室和火山引擎作为语音提供商的支持，我想很有用？
|
||||
|
||||
### UI
|
||||
|
||||
- 新的[教程步骤器](https://airi.moeru.ai/ui/#/story/src-components-misc-steppers-steppers-story-vue?variantId=src-components-misc-steppers-steppers-story-vue-0)、[文件上传](https://airi.moeru.ai/ui/#/story/src-components-form-input-inputfile-story-vue?variantId=default) 和 [文本区域](https://airi.moeru.ai/ui/#/story/src-components-form-textarea-textarea-story-vue?variantId=default) 组件
|
||||
- 颜色问题
|
||||
- [排版改进](https://airi.moeru.ai/ui/#/story/stories-typographysans-story-vue?)
|
||||
|
||||
更多故事可以在 [Roadmap v0.5](https://github.com/moeru-ai/airi/issues/113) 中找到
|
||||
|
||||
## 副任务
|
||||
|
||||
### [Velin](https://github.com/luoling8192/velin)
|
||||
|
||||
自从我们支持了角色卡，在处理模板变量渲染和组件重用时感觉不是很好和流畅...
|
||||
|
||||
如果...
|
||||
|
||||
- 我们可以维护一个组件提示词库，可以用于其他代理或角色扮演应用程序，甚至角色卡？
|
||||
- 例如：
|
||||
- 为魔法和龙拥有中世纪奇幻背景设置
|
||||
- 我们唯一需要做的就是在将世界设置包装在外面时专注于我们新角色的写作
|
||||
- 也许，只有当时间到了夜晚，特殊的提示词才会通过 `if` 和 `if-else` 控制流被注入
|
||||
- 我们可以围绕它做更多事情...
|
||||
- 使用 Vue SFC 或 React JSX，我们可以解析模板并识别 props，在编写提示词时渲染一个用于调试和测试的表单面板
|
||||
- 在单个交互页面中可视化整个 lorebook 和角色卡
|
||||
|
||||
那么为什么我们不制作一个工具来用前端框架如 Vue 或 React 编写 LLM 提示词，也许将此扩展到其他框架和平台？
|
||||
|
||||
这就是我们得到的：[**Velin**](https://github.com/luoling8192/velin)。
|
||||
|
||||
<img class="light" :src="VelinLight" alt="用 Vue.js 编写 LLM 提示词的工具" />
|
||||
<img class="dark" :src="VelinDark" alt="用 Vue.js 编写 LLM 提示词的工具" />
|
||||
|
||||
我们甚至制作了一个用于编辑和实时渲染的游乐场，同时享受 npm 包的生态系统（是的，你可以导入任何包！）。
|
||||
|
||||
<img class="light" :src="VelinPlaygroundLight" alt="用 Vue.js 编写 LLM 提示词的工具" />
|
||||
<img class="dark" :src="VelinPlaygroundDark" alt="用 Vue.js 编写 LLM 提示词的工具" />
|
||||
|
||||
在这里试试：https://velin-dev.netlify.app
|
||||
|
||||
也支持编程 API，Markdown（MDX 正在开发中，支持 MDC），你今天就可以通过 npm 安装它！
|
||||
|
||||
```bash
|
||||
npm install @velin-dev/core
|
||||
```
|
||||
|
||||
好吧...今天就到这里，我希望你们喜欢阅读这篇 DevLog。
|
||||
|
||||
让我们用我们最近在中国杭州参加的活动的更多图片来结束 DevLog：**Demo Day @ 杭州**。
|
||||
|
||||
<img :src="DemoDayHangzhou1" alt="Demo Day @ 杭州" />
|
||||
|
||||
这是我，我与其他参与者分享了 AIRI 项目，我们在那里度过了美好的时光！遇到了许多有才华的开发者、产品设计师和企业家。
|
||||
|
||||
介绍了我今天在这篇 DevLog 中分享的几乎所有内容，还有备受喜爱的 AI VTuber Neuro-sama。
|
||||
|
||||
我用来分享的幻灯片是这样的：
|
||||
|
||||
<img :src="DemoDayHangzhou2" alt="Demo Day @ 杭州" />
|
||||
<img :src="DemoDayHangzhou3" alt="Demo Day @ 杭州" />
|
||||
|
||||
幻灯片本身是完全开源的，你也可以在这里玩玩：[https://talks.ayaka.io/nekoayaka/2025-05-10-airi-how-we-recreated-it/#/1](https://talks.ayaka.io/nekoayaka/2025-05-10-airi-how-we-recreated-it/#/1)
|
||||
|
||||
## 里程碑
|
||||
|
||||
哦...由于这篇 DevLog 也标志着 v0.5.0 的发布，我想提及一些我们在过去几周达到的里程碑：
|
||||
|
||||
- 我们达到了 700 颗星！
|
||||
- 4+ 个新的 issue 贡献者！
|
||||
- Discord 服务器中 72+ 个新的群组成员！
|
||||
- ReLU 角色设计完成！
|
||||
- ReLU 角色建模完成！
|
||||
- 与几家公司就赞助和合作进行了谈判！
|
||||
- [Roadmap v0.5](https://github.com/moeru-ai/airi/issues/113) 完成了 92 个任务
|
||||
- UI
|
||||
- 加载屏幕和教程模块
|
||||
- 多个错误修复，包括加载状态和 Firefox 兼容性问题
|
||||
- 身体
|
||||
- 动作嵌入和来自语义的 RAG，在私有仓库"moeru-ai/motion-gen"中开发
|
||||
- 使用嵌入提供商和 DuckDB WASM 的向量存储和检索
|
||||
- 输入
|
||||
- 修复了 Discord 语音频道语音识别
|
||||
- 输出
|
||||
- 实验性唱歌功能
|
||||
- 工程
|
||||
- 跨项目共享 UnoCSS 配置
|
||||
- "moeru-ai/inventory"中的模型目录
|
||||
- 跨组织的包重组
|
||||
- 资源
|
||||
- 新的角色资源，包括贴纸、UI 元素、VTuber 标志
|
||||
- 语音线选择功能
|
||||
- 角色"Me"和"ReLU"的 Live2D 建模
|
||||
- 社区支持和营销
|
||||
- 日语 README
|
||||
- Plausible 分析集成
|
||||
- 全面的文档
|
||||
|
||||
再见！
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 4.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.9 KiB |
@@ -0,0 +1,110 @@
|
||||
---
|
||||
title: 开发日志 @ 2025.06.08
|
||||
category: DevLog
|
||||
date: 2025-06-08
|
||||
excerpt: |
|
||||
如何让 Live2D 模型跟随鼠标位置，以及在多显示器环境下计算的挑战。
|
||||
preview-cover:
|
||||
light: "@assets('./assets/250608-light.avif')"
|
||||
dark: "@assets('./assets/250608-dark.avif')"
|
||||
---
|
||||
|
||||
大家好，我是 LemonNeko，AIRI 的维护者之一，由我来为大家带来今天的 DevLog：让 AIRI 桌宠的 Live2D 模型可以注视鼠标位置。
|
||||
|
||||
Hello everyone, here's LemonNeko, one of maintainer of AIRI. Today's DevLog is talking about: Let Live2D model of AIRI Tamagotchi to focus position.
|
||||
|
||||
## 思路整理「Chain of Thoughts」
|
||||
|
||||
`<think>`
|
||||
|
||||
首先我们需要了解，在 Live2D 中有两种基础的交互：注视 (focus) 与触碰 (tap)，当我们创建一个 Live2D 画布，模型就会自动注视我们的鼠标位置，头和身体朝向鼠标这一侧。下面是实现后的效果：
|
||||
|
||||
First of all, we need to know, there are two basic interactions of Live2D: **Focus**, and **Tap**, when we create a Live2D canvas, model will auto focus the position of our cursor, head will look at it, like this:
|
||||
|
||||
|
||||
|
||||
但是当鼠标离开网页内容后，Live2D 就不再会知道鼠标的位置在哪了，所以我们需要手动告诉它鼠标在哪。
|
||||
|
||||
But, if the cursor is out of web page, Live2D won't know the position of our cursor, so we need to tell the Live2D engine where is our cursor.
|
||||
|
||||
为了告诉 Live2D 鼠标的位置，我们需要利用 Tauri 的原生代码调用能力来调用 Windows API 和 macOS API，~~写一大堆 unsafe~~ 来取得鼠标在整块屏幕上的位置与窗口本身的位置，最后进行一些简单的计算，得到鼠标与窗口的相对位置。
|
||||
|
||||
To tell the position of cursor to Live2D, we need to use native code calling ability of Tauri, and get the position of cursor and window frame. Then we can calculate the relative position of cursor to window.
|
||||
|
||||
`</think>`
|
||||
|
||||
## 计算鼠标与窗口的相对位置<br>Calculate the relative position of cursor to window
|
||||
|
||||
假设这是我们的屏幕：
|
||||
|
||||
For example, we have a screen like this:
|
||||
|
||||
|
||||
|
||||
蓝色框是屏幕，粉色是 AIRI 的窗口，紫色箭头是鼠标，我们定义：
|
||||
|
||||
The blue box is the screen, the pink box is the AIRI window, the purple arrow is the cursor, we define:
|
||||
|
||||
- 屏幕高宽为：`A x B`<br>The screen size is: `A x B`
|
||||
- AIRI 窗口左上角位置为：`(E, F)`<br>The position of AIRI window is: `(E, F)`
|
||||
- AIRI 窗口大小为：`C x D`<br>The size of AIRI window is: `C x D`
|
||||
- 鼠标位置是：`G, H`<br>The position of cursor is: `G, H`
|
||||
|
||||
那么鼠标的位置在 AIRI 窗口中的位置应当是：`(G - E, H - F)`
|
||||
|
||||
Then the relative position of cursor to window is: `(G - E, H - F)`
|
||||
|
||||
似乎非常简单啊，那在代码中写出来的话，应该会是这样子的：
|
||||
|
||||
It seems very simple, right? Then let's write the code.
|
||||
|
||||
```typescript
|
||||
const live2dFocusAt = ref({ x: innerWidth / 2, y: innerHeight / 2 }) // initial position
|
||||
|
||||
listen('tauri-app:window-click-through:mouse-location-and-window-frame', (event: { payload: [Point, WindowFrame] }) => {
|
||||
const [mouseLocation, windowFrame] = event.payload
|
||||
|
||||
live2dFocusAt.value = {
|
||||
x: mouseLocation.x - windowFrame.origin.x,
|
||||
y: mouseLocation.y - windowFrame.origin.y,
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
`live2dFocusAt` 是要传递给 Live2D 模型的坐标数据。
|
||||
|
||||
`live2dFocusAt` is the coordinate data that will be passed to the Live2D model.
|
||||
|
||||
## 手动设置模型的注视点<br>Set the focus point of Live2D model manually
|
||||
|
||||
在代码里就是这样，把我们在上面定义的 `live2dFocusAt` 传递给 Live2D 模型：
|
||||
|
||||
We can set the focus point of Live2D model manually by passing the `live2dFocusAt` to the Live2D model.
|
||||
|
||||
```typescript
|
||||
const model = ref(Live2DModel.from('url', { autoInteract: false }))
|
||||
|
||||
watch(live2dFocusAt, (point) => {
|
||||
model.value.focus(point)
|
||||
})
|
||||
```
|
||||
|
||||
## 多平台适配<br>Multi-platform support
|
||||
|
||||
很不幸的是，事情并没有我想象的那么简单，上面提到的得到鼠标与窗口相对位置的思路在 Windows 上是工作的，来到了 macOS 上就发现不起作用了，因为 macOS 的坐标系中原点在左下角，**Y 轴向上**，与 Windows 相反，但是在 Safari 浏览器中，坐标系的原点在左上角，**Y 轴向下**，所以我们的鼠标位置在 macOS 上应该表示为 `(G - E, D - H + F)`。
|
||||
|
||||
Unfortunately, the story is not as simple as I thought. The idea of getting the relative position of cursor to window works on Windows, but it doesn't work on macOS. In macOS, the origin of the coordinate system is at the bottom left corner, **Y axis is up**, which is opposite to Windows. But in Safari browser, the origin of the coordinate system is at the top left corner, **Y axis is down**, so the cursor position on macOS should be represented as `(G - E, D - H + F)`.
|
||||
|
||||
## 阅读更多<br>Read more
|
||||
|
||||
好了，我们实现了 Live2D 模型在 Tauri 上跟随窗口之外的鼠标位置，这就是本次 DevLog 的全部内容，以下是我在实现过程中查阅的资料，欢迎详细阅读以及讨论：
|
||||
|
||||
In this DevLog, we learned how to get the relative position of cursor to window, and how to set the focus point of Live2D model manually. If you want to know more about the implementation details, you can check the [source code](https://github.com/moeru-ai/airi/pull/194) of this PR.
|
||||
|
||||
- [手动配置模型的交互 - pixi-live2d-display](https://github.com/guansss/pixi-live2d-display/wiki/Complete-Guide#manually-1 "手动配置模型的交互 - pixi-live2d-display")
|
||||
- [Win32 API: GetCursorPos](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getcursorpos "GetCursorPos")
|
||||
- [Win32 API: GetWindowRect](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getwindowrect "GetWindowRect")
|
||||
- [macOS API: `NSWindow.frame`](https://developer.apple.com/documentation/appkit/nswindow/frame "NSWindow.frame")
|
||||
- [macOS API: `NSEvent.mouseLocation`](https://developer.apple.com/documentation/appkit/nsevent/mouselocation "NSEvent.mouseLocation")
|
||||
|
||||
> 封面图片由 [@Rynco Maekawa](https://github.com/lynzrand) 提供<br>Cover image by [@Rynco Maekawa](https://github.com/lynzrand)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user