diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index 98e8b390c..449677ae5 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -200,6 +200,7 @@ "@types/culori": "catalog:", "@types/d3": "catalog:", "@types/hast": "catalog:", + "@types/mdast": "catalog:", "@types/splitpanes": "catalog:", "@types/unist": "catalog:", "@types/ws": "catalog:", diff --git a/packages/stage-ui/src/components/markdown/markdown-renderer.browser.test.ts b/packages/stage-ui/src/components/markdown/markdown-renderer.browser.test.ts index 058af1817..7107cec8b 100644 --- a/packages/stage-ui/src/components/markdown/markdown-renderer.browser.test.ts +++ b/packages/stage-ui/src/components/markdown/markdown-renderer.browser.test.ts @@ -4,16 +4,16 @@ import { defineComponent, onMounted, ref, useTemplateRef } from 'vue' import MarkdownRenderer from './markdown-renderer.vue' -const markdown = `### Roof Leak in Server Room - -*Anime style virtual AI girl waking up in a server room and noticing water from the ceiling.* - -> **Sharing**: Sending a quick sketch to our chat history...` - const MarkdownMountProbe = defineComponent({ components: { MarkdownRenderer, }, + props: { + markdown: { + type: String, + required: true, + }, + }, emits: { mountedHtml: (html: string) => typeof html === 'string', }, @@ -25,7 +25,6 @@ const MarkdownMountProbe = defineComponent({ }) return { - markdown, root, } }, @@ -36,22 +35,26 @@ const MarkdownMountProbe = defineComponent({ `, }) -const MarkdownHarness = defineComponent({ - components: { - MarkdownMountProbe, - }, - setup() { - const mountedHtml = ref('') +function createMarkdownHarness(markdown: string, label: string) { + return defineComponent({ + components: { + MarkdownMountProbe, + }, + setup() { + const mountedHtml = ref('') - return { - mountedHtml, - } - }, - template: ` - - {{ mountedHtml }} - `, -}) + return { + label, + markdown, + mountedHtml, + } + }, + template: ` + + {{ mountedHtml }} + `, + }) +} describe('markdown renderer initial content', () => { it('renders basic Markdown before mounted layout code reads the element', async () => { @@ -62,8 +65,31 @@ describe('markdown renderer initial content', () => { // The content jumped to its real height when the animation released its fixed height. // // We fixed this by rendering basic Markdown synchronously before optional rich processing. - const screen = await render(MarkdownHarness) + const markdown = `### Roof Leak in Server Room + +*Anime style virtual AI girl waking up in a server room and noticing water from the ceiling.* + +> **Sharing**: Sending a quick sketch to our chat history...` + const screen = await render(createMarkdownHarness(markdown, 'initial-markdown-html')) await expect.element(screen.getByLabelText('initial-markdown-html')).toHaveTextContent('

Roof Leak in Server Room

') }) + + // https://github.com/moeru-ai/airi/discussions/2239 + it('renders double-dollar math without consuming currency for Issue #2239', async () => { + // ROOT CAUSE: + // + // Single-dollar math can pair two currency signs. The rendered component + // then displays normal prose as a formula. + // + // We fixed this by keeping single dollars as text and using double dollars + // for inline math. + const markdown = 'Price is $5 and cost is $10.\n\nThe result is $$x^2$$.' + const screen = await render(createMarkdownHarness(markdown, 'chat-math-html')) + const html = screen.getByLabelText('chat-math-html') + + await expect.element(html).toHaveTextContent('Price is $5 and cost is $10.') + await expect.element(html).toHaveTextContent(' = () => (tree) => { + visit(tree, 'code', (node, index, parent) => { + const language = node.lang?.toLowerCase() + if (index === undefined || !parent || !isChatMathFenceLanguage(language)) + return + + // `math` is the canonical remark-math fence. It must be filtered from + // Shiki, but only `latex` and `tex` fences need conversion here. + if (language === 'math') + return + + const rows = node.value + .split(/\r?\n/) + .map(row => row.trim()) + .filter(Boolean) + + if (rows.length === 0) { + // Streaming can expose a chat math fence before its first formula row. + // Consume the marker instead of rendering an empty code block. + parent.children.splice(index, 1) + return [SKIP, index] + } + + const meta = node.meta?.toLowerCase().split(/\s+/).filter(Boolean) ?? [] + const values = meta.includes('block') ? [node.value.trim()] : rows + const mathNodes: RootContent[] = values.map(value => ({ + type: 'code', + lang: 'math', + meta: null, + value, + })) + + parent.children.splice(index, 1, ...mathNodes) + // Tell the unist visitor to skip the newly inserted nodes and resume at + // the index immediately after them, so each row is visited only once. + return [SKIP, index + mathNodes.length] + }) +} + +/** + * Defines the math syntax for AIRI chat Markdown. + * + * A single dollar sign stays text, and `$$...$$` defines inline math. A + * `latex` or `tex` fence contains one formula per non-empty row. The `block` + * meta value keeps the fence intact. + */ +export const chatMathPreset = { + plugins: [ + [remarkMath, { singleDollarTextMath: false }], + remarkChatMath, + ], +} satisfies Preset diff --git a/packages/stage-ui/src/composables/markdown.test.ts b/packages/stage-ui/src/composables/markdown.test.ts new file mode 100644 index 000000000..ed4fc7971 --- /dev/null +++ b/packages/stage-ui/src/composables/markdown.test.ts @@ -0,0 +1,378 @@ +import { describe, expect, it } from 'vitest' + +import { useMarkdown } from './markdown' + +function mathNodeCount(html: string): number { + return html.match(/ { + // https://github.com/moeru-ai/airi/discussions/2239 + it('renders each LaTeX fence line as display math for Issue #2239', async () => { + // ROOT CAUSE: + // + // Markdown treats a LaTeX fence as source code. The old fix then tried to + // infer formula boundaries from relations and operators in each line. + // + // We define a chat syntax instead. Each non-empty line in a `latex` or + // `tex` fence is one display formula. + const markdown = [ + '```latex', + String.raw`\frac{d}{dx}(c)=0`, + String.raw`\frac{d}{dx}(x^n)=n x^{n-1}`, + String.raw`\frac{d}{dx}(e^x)=e^x`, + String.raw`\int c\,dx = cx + C`, + String.raw`\int x^n\,dx = \frac{x^{n+1}}{n+1}+C`, + String.raw`\int \sin x\,dx = -\cos x + C`, + '```', + ].join('\n') + + const { process, processSync } = useMarkdown() + const initialHtml = processSync(markdown) + const html = await process(markdown) + + expect(mathNodeCount(initialHtml)).toBe(6) + expect(html).not.toContain('class="shiki') + expect(mathNodeCount(html)).toBe(6) + expect(html).toContain('\\frac{d}{dx}(c)=0') + expect(html).toContain('\\frac{d}{dx}(x^n)=n x^{n-1}') + expect(html).toContain('\\frac{d}{dx}(e^x)=e^x') + expect(html).toContain('\\int c\\,dx = cx + C') + expect(html).toContain('\\int x^n\\,dx = \\frac{x^{n+1}}{n+1}+C') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3818349198 + it('renders formula-list rows without a relation allowlist', () => { + // ROOT CAUSE: + // + // The old splitter required a known relation on every line. Commands such + // as `\in`, `\notin`, and `\subseteq` form an open set. Expressions can + // also be independent formulas without any relation. + // + // We fixed this by making physical rows part of the `latex` fence syntax. + const markdown = [ + '```latex', + String.raw`x \in A`, + String.raw`y \notin B`, + String.raw`A \subseteq C`, + String.raw`\sin x`, + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(4) + }) + + // https://github.com/moeru-ai/airi/discussions/2239 + it('supports the explicit tex rows alias with blank rows and CRLF for Issue #2239', () => { + const markdown = [ + '```tex rows', + String.raw`\int c\,dx = cx + C`, + '', + String.raw`\int e^x\,dx = e^x + C`, + '```', + ].join('\r\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(2) + }) + + // https://github.com/moeru-ai/airi/pull/2328 + it('keeps a latex block as one formula without inspecting its rows', () => { + // ROOT CAUSE: + // + // Content heuristics split any rows that looked like complete equations. + // This ignored the fence mode and changed macro scope and layout. + // + // We fixed this by making `block` the only boundary decision. + const markdown = [ + '```latex block', + 'x = 1', + 'y = 2', + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).toContain('x = 1\ny = 2') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3819441402 + it('keeps syntax highlighting when a math fence has metadata', async () => { + // ROOT CAUSE: + // + // The rich pipeline loaded the complete fence info string as a Shiki + // language. For example, it tried to load `latex block`. Shiki rejected + // that name, and the fallback removed highlighting from unrelated code. + // + // The language loader must read only the first info-string token. + const markdown = [ + '```latex block', + 'x = 1', + 'y = 2', + '```', + '', + '```typescript', + 'const answer = 42', + '```', + ].join('\n') + + const html = await useMarkdown().process(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).toContain('class="shiki') + expect(html).toContain('>const') + expect(html).toContain('> answer') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3819513359 + it('keeps syntax highlighting when a message has a math fence', async () => { + // ROOT CAUSE: + // + // A `math` fence is a remark-math marker, not a Shiki language. Loading + // it in the rich pipeline made Shiki reject the processor and removed + // highlighting from every code block in the message. + // + // Chat math fence languages must not enter the Shiki language list. + const markdown = [ + '```math', + String.raw`\begin{aligned}`, + String.raw`x &= 1 \\`, + String.raw`y &= 2`, + String.raw`\end{aligned}`, + '```', + '', + '```typescript', + 'const answer = 42', + '```', + ].join('\n') + + const html = await useMarkdown().process(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).toContain('class="shiki') + expect(html).toContain('>const') + expect(html).toContain('> answer') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3819733711 + it('consumes empty chat math fences without disabling syntax highlighting for PR #2328', async () => { + // ROOT CAUSE: + // + // Empty `latex` and `tex` fences were excluded from Shiki language + // loading but remained code nodes, so streaming displayed blank code + // blocks before the first formula row arrived. + // + // Chat math fences must be consumed even before streaming adds a formula. + const markdown = [ + '```latex', + '```', + '', + '```tex', + ' ', + '```', + '', + '```typescript', + 'const answer = 42', + '```', + ].join('\n') + + const html = await useMarkdown().process(markdown) + + expect(mathNodeCount(html)).toBe(0) + expect(html).not.toContain('language-latex') + expect(html).not.toContain('language-tex') + expect(html).toContain('class="shiki') + expect(html).toContain('>const') + expect(html).toContain('> answer') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3819568323 + it('loads a code language from a fence nested in a blockquote', async () => { + // ROOT CAUSE: + // + // A source-text regex only found top-level fences. Remark still parsed a + // fence inside a blockquote, but Shiki did not preload its language and + // made the complete rich pipeline fall back. + // + // Language discovery must use the same Markdown AST as rendering. + const markdown = [ + '> ```typescript', + '> const answer = 42', + '> ```', + ].join('\n') + + const html = await useMarkdown().process(markdown) + + expect(html).toContain('
') + expect(html).toContain('class="shiki') + expect(html).toContain('>const') + expect(html).toContain('> answer') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3812513778 + it('keeps command arguments together in a latex block', () => { + const markdown = [ + '```latex block', + String.raw`\frac{a=b}`, + String.raw`{c=d}`, + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).not.toContain(' { + const markdown = [ + '```tex block', + String.raw`\left(x=1`, + String.raw`\right)=y`, + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).not.toContain('katex-error') + expect(html).toContain('\\left(x=1\n\\right)=y') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3818173297 + it('keeps optional command arguments together in a latex block', () => { + const markdown = [ + '```latex block', + String.raw`y = \sqrt`, + '[3]{x = z}', + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).not.toContain('katex-error') + expect(html).toContain('y = \\sqrt\n[3]{x = z}') + }) + + // https://github.com/moeru-ai/airi/pull/2328#discussion_r3818250572 + it('keeps scripts together in a latex block', () => { + const markdown = [ + '```latex block', + String.raw`S = \sum`, + String.raw`_{i=1}^{n} i = n(n+1)/2`, + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).not.toContain('katex-error') + expect(html).toContain('S = \\sum\n_{i=1}^{n} i = n(n+1)/2') + }) + + it('keeps a multiline math fence as one formula', () => { + const markdown = [ + '```math', + String.raw`\begin{aligned}`, + String.raw`f(x) &= x^2 \\`, + String.raw`f'(x) &= 2x`, + String.raw`\end{aligned}`, + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).not.toContain(' { + const markdown = [ + '```tex block', + String.raw`\newcommand{\foo}{x=1}`, + String.raw`\foo=2`, + '```', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(1) + expect(html).not.toContain(' { + // ROOT CAUSE: + // + // Single-dollar math pairs normal currency signs before Markdown can know + // whether the content is prose or a formula. Word-based recovery rules + // then conflict with valid variables and units. + // + // We fixed this by disabling single-dollar math for the chat syntax. + const markdown = [ + 'Price is $5 and cost is $10.', + 'Prices are $5 and $10.', + 'Tickets cost $5-$10.', + 'Tickets cost $5 to $10.', + 'The old formula syntax is $5 + x$.', + 'Values are $5$ and $10$.', + ].join('\n\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(0) + expect(html).toContain('Price is $5 and cost is $10.') + expect(html).toContain('Prices are $5 and $10.') + expect(html).toContain('Tickets cost $5-$10.') + expect(html).toContain('Tickets cost $5 to $10.') + expect(html).toContain('The old formula syntax is $5 + x$.') + expect(html).toContain('Values are $5$ and $10$.') + }) + + it('renders double-dollar inline math', () => { + const html = useMarkdown().processSync('The result is $$5 + x$$.') + + expect(mathNodeCount(html)).toBe(1) + expect(html).toContain('5 + x') + }) + + it('renders each display-math block as one formula', () => { + const markdown = [ + '$$', + 'x = 1', + '$$', + '', + '$$', + String.raw`y \in A`, + '$$', + ].join('\n') + + const html = useMarkdown().processSync(markdown) + + expect(mathNodeCount(html)).toBe(2) + }) + + it('keeps invalid formula source visible', () => { + const html = useMarkdown().processSync('$$\\notacommand{x}$$') + + expect(html).toContain('\\notacommand{x}') + expect(html).not.toContain(' { + const html = useMarkdown().processSync(String.raw`$$\href{javascript:alert(1)}{x}$$`) + + expect(html).toContain(String.raw`\href{javascript:alert(1)}{x}`) + expect(html).not.toContain(' const processorCache = new Map>() -const langRegex = /```(.{2,})\s/g +// The rich path parses once for language discovery so info strings and fences +// inside blockquotes or nested lists follow Remark rules instead of a regex. +const languageParser = unified() + .use(RemarkParse) + .use(chatMathPreset) function extractLangs(markdown: string): BundledLanguage[] { - const matches = markdown.matchAll(langRegex) + const tree = languageParser.parse(markdown) const langs = new Set() langs.add('python') - for (const match of matches) { - if (match[1]) - langs.add(match[1] as BundledLanguage) - } + visit(tree, 'code', (node) => { + if (node.lang && !isChatMathFenceLanguage(node.lang)) + langs.add(node.lang as BundledLanguage) + }) return [...langs] } @@ -61,7 +67,7 @@ async function createProcessor(langs: BundledLanguage[]): Promise { export function useMarkdown() { const fallbackProcessor = unified() .use(RemarkParse) - .use(remarkMath) + .use(chatMathPreset) .use(RemarkRehype) .use(measuredKatex, { output: 'mathml' }) .use(RehypeStringify) diff --git a/packages/stage-ui/src/stores/chat/session-store.test.ts b/packages/stage-ui/src/stores/chat/session-store.test.ts index c2e0a735d..71780f183 100644 --- a/packages/stage-ui/src/stores/chat/session-store.test.ts +++ b/packages/stage-ui/src/stores/chat/session-store.test.ts @@ -606,6 +606,21 @@ describe('chat-session-store · cloud deletion', () => { }) describe('chat-session-store · active card prompt edits', () => { + // https://github.com/moeru-ai/airi/discussions/2239 + it('adds the AIRI chat math syntax to the system message for Issue #2239', async () => { + const store = useChatSessionStore() + await store.initialize() + + const content = store.messages[0]?.content + + expect(content).toContain('Use $$...$$ for inline math.') + expect(content).toContain('Use a separate multiline $$ block for each display equation.') + expect(content).toContain('Use a latex fence for a list of independent one-line equations.') + expect(content).toContain('Use a math fence for one multiline equation or LaTeX environment.') + expect(content).toContain('Do not use single dollar signs as math delimiters.') + expect(content).not.toContain('eg: $ x^3 $') + }) + // ROOT CAUSE: // // Editing the active card updates `systemPrompt`, but the session store only diff --git a/packages/stage-ui/src/stores/chat/session-store.ts b/packages/stage-ui/src/stores/chat/session-store.ts index 3eef6682c..64f2ac761 100644 --- a/packages/stage-ui/src/stores/chat/session-store.ts +++ b/packages/stage-ui/src/stores/chat/session-store.ts @@ -124,7 +124,13 @@ export const useChatSessionStore = defineStore('chat-session', () => { // I know this nu uh, better than loading all language on rehypeShiki const codeBlockSystemPrompt = '- For any programming code block, always specify the programming language that supported on @shikijs/rehype on the rendered markdown, eg. ```python ... ```\n' - const mathSyntaxSystemPrompt = '- For any math equation, use LaTeX format, eg: $ x^3 $, always escape dollar sign outside math equation\n' + const mathSyntaxSystemPrompt = `${[ + '- Use $$...$$ for inline math.', + '- Use a separate multiline $$ block for each display equation.', + '- Use a latex fence for a list of independent one-line equations.', + '- Use a math fence for one multiline equation or LaTeX environment.', + '- Do not use single dollar signs as math delimiters.', + ].join('\n')}\n` function getCurrentUserId() { return userId.value || 'local' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 707a9b91c..3dbe0106c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -450,6 +450,9 @@ catalogs: '@types/markdown-it': specifier: ^14.1.2 version: 14.1.2 + '@types/mdast': + specifier: ^4.0.4 + version: 4.0.4 '@types/node': specifier: ^24.12.2 version: 24.12.2 @@ -4601,6 +4604,9 @@ importers: '@types/hast': specifier: 'catalog:' version: 3.0.4 + '@types/mdast': + specifier: 'catalog:' + version: 4.0.4 '@types/splitpanes': specifier: 'catalog:' version: 2.2.6 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ffa3d6597..fc8c8fefd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -185,6 +185,7 @@ catalog: '@types/hast': ^3.0.4 '@types/ioredis-mock': ^8.2.8 '@types/markdown-it': ^14.1.2 + '@types/mdast': ^4.0.4 '@types/node': ^24.12.2 '@types/nprogress': ^0.2.3 '@types/pg': ^8.20.0