import { describe, expect, it } from 'vitest' import { createStreamingCategorizer } from './response-categoriser' describe('createStreamingCategorizer', () => { it('should handle pure speech without tags', () => { const categorizer = createStreamingCategorizer() const text = 'Hello, world! This is a test.' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe(text) expect(result.reasoning).toBe('') expect(result.segments).toEqual([]) }) it('should filter out reasoning tags from speech', () => { const categorizer = createStreamingCategorizer() const text = 'Hello thinking about this world!' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('thinking about this') expect(result.segments).toHaveLength(1) expect(result.segments[0].tagName).toBe('reasoning') expect(result.segments[0].content).toBe('thinking about this') }) it('should handle multiple reasoning tags', () => { const categorizer = createStreamingCategorizer() const text = 'Start first thought middle second thought end' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('Start middle end') expect(result.reasoning).toBe('first thought\n\nsecond thought') expect(result.segments).toHaveLength(2) }) it('should filter incomplete tags from speech during streaming', () => { const categorizer = createStreamingCategorizer() // Stream incomplete tag categorizer.consume('Hello thinking') const filtered1 = categorizer.filterToSpeech('Hello thinking', 0) // Before tag closes, everything should be filtered (tag is incomplete) expect(filtered1).toBe('') // Complete the tag - the important thing is the final result is correct categorizer.consume(' about this world!') const result = categorizer.end() expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('thinking about this') }) it('should handle tags split across chunks', () => { const categorizer = createStreamingCategorizer() const chunks = [ 'Hello <', 'reasoning', '>thinking', ' about this', ' world!', ] for (const chunk of chunks) { categorizer.consume(chunk) } const result = categorizer.end() // Final result should correctly categorize the complete text expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('thinking about this') }) it('should handle nested tags', () => { const categorizer = createStreamingCategorizer() const text = 'Start outer inner more outer end' categorizer.consume(text) const result = categorizer.end() expect(result.segments.length).toBeGreaterThan(0) expect(result.speech).toBe('Start end') expect(result.reasoning).toContain('inner') }) it('should correctly identify speech positions with isSpeechAt', () => { const categorizer = createStreamingCategorizer() const text = 'Hello thinking world!' categorizer.consume(text) // Position 0-5: "Hello " - should be speech expect(categorizer.isSpeechAt(0)).toBe(true) expect(categorizer.isSpeechAt(5)).toBe(true) // Position 6-36: inside tag (including the tag itself) - should not be speech // "Hello " = 6 chars, "" starts at 6, "" ends at 36 expect(categorizer.isSpeechAt(6)).toBe(false) expect(categorizer.isSpeechAt(20)).toBe(false) expect(categorizer.isSpeechAt(36)).toBe(false) // Position 37+: " world!" - should be speech (after closing tag) expect(categorizer.isSpeechAt(37)).toBe(true) expect(categorizer.isSpeechAt(43)).toBe(true) }) it('should handle tags with special tokens like emotes inside reasoning', () => { const categorizer = createStreamingCategorizer() // Special tokens like <|ACT {"emotion":{"name":"happy","intensity":1}}|> and <|DELAY:1|> should be included in reasoning const text = 'Hello thinking <|ACT {"emotion":{"name":"happy","intensity":1}}|> about this <|DELAY:1|> world!' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('thinking <|ACT {"emotion":{"name":"happy","intensity":1}}|> about this <|DELAY:1|>') }) it('should handle tag with special tokens in between', () => { const categorizer = createStreamingCategorizer() // Special tokens like <|ACT {"emotion":{"name":"curious","intensity":1}}|> should be included within reasoning tags const text = 'Hello thinking <|ACT {"emotion":{"name":"curious","intensity":1}}|> about this <|DELAY:2|> and that world!' categorizer.consume(text) const result = categorizer.end() // Log what was recognized // eslint-disable-next-line no-console console.log({ input: text, reasoning: result.reasoning, segmentContent: result.segments[0]?.content, segmentsFound: result.segments.length, speech: result.speech, tagName: result.segments[0]?.tagName, }) // Verify tag is recognized expect(result.segments).toHaveLength(1) expect(result.segments[0].tagName).toBe('think') // Verify special tokens are preserved in segment content expect(result.segments[0].content).toContain('<|ACT {"emotion":{"name":"curious","intensity":1}}|>') expect(result.segments[0].content).toContain('<|DELAY:2|>') // Verify special tokens are in reasoning output expect(result.reasoning).toContain('<|ACT {"emotion":{"name":"curious","intensity":1}}|>') expect(result.reasoning).toContain('<|DELAY:2|>') // Verify speech excludes the reasoning content expect(result.speech).toBe('Hello world!') expect(result.speech).not.toContain('<|ACT {"emotion":{"name":"curious","intensity":1}}|>') expect(result.speech).not.toContain('<|DELAY:2|>') expect(result.reasoning).toBe('thinking <|ACT {"emotion":{"name":"curious","intensity":1}}|> about this <|DELAY:2|> and that') }) it('should handle with special tokens like <|ACT {"emotion":{"name":"happy","intensity":1}}|> in between', () => { const categorizer = createStreamingCategorizer() // Testing the exact scenario: ...<|Special in between|>... const text = 'Hello thinking <|ACT {"emotion":{"name":"happy","intensity":1}}|> about this <|DELAY:1|> and that world!' categorizer.consume(text) const result = categorizer.end() // Log what was recognized // eslint-disable-next-line no-console console.log({ input: text, reasoning: result.reasoning, segmentContent: result.segments[0]?.content, segmentsFound: result.segments.length, speech: result.speech, tagName: result.segments[0]?.tagName, }) // Verify tag is recognized expect(result.segments).toHaveLength(1) expect(result.segments[0].tagName).toBe('think') // Verify special tokens are preserved in segment content expect(result.segments[0].content).toContain('<|ACT {"emotion":{"name":"happy","intensity":1}}|>') expect(result.segments[0].content).toContain('<|DELAY:1|>') // Verify special tokens are in reasoning output expect(result.reasoning).toContain('<|ACT {"emotion":{"name":"happy","intensity":1}}|>') expect(result.reasoning).toContain('<|DELAY:1|>') // Verify speech excludes the reasoning content expect(result.speech).toBe('Hello world!') expect(result.speech).not.toContain('<|ACT {"emotion":{"name":"happy","intensity":1}}|>') expect(result.speech).not.toContain('<|DELAY:1|>') expect(result.reasoning).toBe('thinking <|ACT {"emotion":{"name":"happy","intensity":1}}|> about this <|DELAY:1|> and that') }) it('should handle any tag name as reasoning', () => { const categorizer = createStreamingCategorizer() const text = 'Hello secret thoughts world!' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('secret thoughts') expect(result.segments[0].tagName).toBe('think') }) it('should handle multiple tags with speech in between', () => { const categorizer = createStreamingCategorizer() const text = 'First thought1 Second thought2 Third' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('First Second Third') expect(result.reasoning).toBe('thought1\n\nthought2') expect(result.segments).toHaveLength(2) }) it('should return empty speech when only tags present', () => { const categorizer = createStreamingCategorizer() const text = 'only reasoning' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('') expect(result.reasoning).toBe('only reasoning') }) it('should handle streaming with incremental categorization', () => { const categorizer = createStreamingCategorizer() // Stream in small chunks categorizer.consume('Hello ') expect(categorizer.getCurrent()?.speech).toBe('Hello ') categorizer.consume('') categorizer.consume('thinking') // Tag not closed yet, but should still categorize const midResult = categorizer.getCurrent() expect(midResult).not.toBeNull() categorizer.consume('') categorizer.consume(' world!') const result = categorizer.end() expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('thinking') }) it('should call onSegment callback when segments are detected', () => { const segments: Array<{ content: string, tagName: string }> = [] const categorizer = createStreamingCategorizer(undefined, (segment) => { segments.push({ content: segment.content, tagName: segment.tagName }) }) const text = 'Hello thought1 thought2 world!' // Stream the text - segments are emitted when they complete categorizer.consume(text) categorizer.end() // Check final result has both segments const result = categorizer.end() expect(result.segments.length).toBeGreaterThanOrEqual(2) expect(result.segments.some(s => s.tagName === 'reasoning')).toBe(true) expect(result.segments.some(s => s.tagName === 'think')).toBe(true) // onSegment is called when segments complete during streaming // At minimum, we should have segments detected in the final result expect(result.segments.length).toBeGreaterThanOrEqual(2) }) it('should handle getCurrentPosition correctly', () => { const categorizer = createStreamingCategorizer() expect(categorizer.getCurrentPosition()).toBe(0) categorizer.consume('Hello') expect(categorizer.getCurrentPosition()).toBe(5) categorizer.consume(' world!') expect(categorizer.getCurrentPosition()).toBe(12) }) it('should handle edge case: tag at start', () => { const categorizer = createStreamingCategorizer() const text = 'thinkingHello world!' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('thinking') }) it('should handle edge case: tag at end', () => { const categorizer = createStreamingCategorizer() const text = 'Hello world!thinking' categorizer.consume(text) const result = categorizer.end() expect(result.speech).toBe('Hello world!') expect(result.reasoning).toBe('thinking') }) it('should handle malformed tags gracefully', () => { const categorizer = createStreamingCategorizer() const text = 'Hello thinking world!' categorizer.consume(text) const result = categorizer.end() // Unclosed tag should be treated as speech expect(result.speech).toContain('Hello') expect(result.segments.length).toBe(0) }) it('should filter speech correctly when tag closes mid-chunk', () => { const categorizer = createStreamingCategorizer() // Stream incomplete tag categorizer.consume('Hello thinking') let filtered = categorizer.filterToSpeech('Hello thinking', 0) expect(filtered).toBe('') // Tag not closed, filter everything // Stream closing tag categorizer.consume('') filtered = categorizer.filterToSpeech('', 25) expect(filtered).toBe('') // This is the closing tag itself // Stream speech after categorizer.consume(' world!') filtered = categorizer.filterToSpeech(' world!', 38) expect(filtered).toBe(' world!') const result = categorizer.end() expect(result.speech).toBe('Hello world!') }) })