nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

style: lint

Browse Source
This commit is contained in:
Neko Ayaka
2026-08-26 19:49:58 +08:00
parent e60a04a4ec
commit 98f40d7d0b
1625 changed files with 75216 additions and 75203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/testing-audio/README.md
+1 -1
View File
@@ -45,13 +45,13 @@ describe('audio input pipeline', () => {
return undefined
return {
captureFormat: 'wav',
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
],
packages/testing-audio/cases/long-leading-silence/case.audio.test.ts
+2 -2
View File
@@ -19,13 +19,13 @@ describe('audio input pipeline', () => {
return undefined
return {
captureFormat: 'wav',
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
],
@@ -64,8 +64,8 @@ describe('audio input pipeline', () => {
return undefined
return {
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
captureFormat: 'pcm',
provider: aliyunNlsAsr({ accessKeyId, accessKeySecret, appKey, provider }),
}
}),
],
packages/testing-audio/cases/shared/configurations/module-consciousness.ts
+3 -3
View File
@@ -9,7 +9,7 @@ export interface ConsciousnessModuleConfiguration {
provider: ProviderConfiguration
}
type ConsciousnessModuleResolver = (context: AudioInputPreflightContext) => ConsciousnessModuleConfiguration | undefined | Promise<ConsciousnessModuleConfiguration | undefined>
type ConsciousnessModuleResolver = (context: AudioInputPreflightContext) => ConsciousnessModuleConfiguration | Promise<ConsciousnessModuleConfiguration | undefined> | undefined
/** Configures the consciousness module with the LLM Provider selected by one case. */
export function configureModuleConsciousness(resolve: ConsciousnessModuleResolver): AudioInputPreflightCallback {
@@ -21,13 +21,13 @@ export function configureModuleConsciousness(resolve: ConsciousnessModuleResolve
await configureProvider(context.runtime, configuration.provider)
await configureActiveCardModules(context.runtime, {
consciousness: {
provider: configuration.provider.id,
model: configuration.provider.model,
provider: configuration.provider.id,
},
})
await configureStorage(context.runtime, {
'settings/consciousness/active-provider': configuration.provider.id,
'settings/consciousness/active-model': configuration.provider.model,
'settings/consciousness/active-provider': configuration.provider.id,
})
}
}
packages/testing-audio/cases/shared/configurations/module-hearing.ts
+3 -3
View File
@@ -14,7 +14,7 @@ export interface HearingModuleConfiguration {
provider: ProviderConfiguration
}
type HearingModuleResolver = (context: AudioInputPreflightContext) => HearingModuleConfiguration | undefined | Promise<HearingModuleConfiguration | undefined>
type HearingModuleResolver = (context: AudioInputPreflightContext) => HearingModuleConfiguration | Promise<HearingModuleConfiguration | undefined> | undefined
/** Configures the hearing module with the ASR Provider selected by one case. */
export function configureModuleHearing(resolve: HearingModuleResolver): AudioInputPreflightCallback {
@@ -25,9 +25,9 @@ export function configureModuleHearing(resolve: HearingModuleResolver): AudioInp
await configureProvider(context.runtime, configuration.provider)
const settings: Record<string, string> = {
'settings/hearing/active-provider': configuration.provider.id,
'settings/hearing/active-model': configuration.provider.model,
'settings/audio/input/enabled': String(configuration.microphoneEnabled ?? false),
'settings/hearing/active-model': configuration.provider.model,
'settings/hearing/active-provider': configuration.provider.id,
}
if (context.runtime.target === 'electron') {
packages/testing-audio/cases/shared/configurations/module-speech.ts
+4 -4
View File
@@ -12,7 +12,7 @@ export interface SpeechModuleConfiguration {
voice: string
}
type SpeechModuleResolver = (context: AudioInputPreflightContext) => SpeechModuleConfiguration | undefined | Promise<SpeechModuleConfiguration | undefined>
type SpeechModuleResolver = (context: AudioInputPreflightContext) => Promise<SpeechModuleConfiguration | undefined> | SpeechModuleConfiguration | undefined
/** Configures the speech module with the TTS Provider selected by one case. */
export function configureModuleSpeech(resolve: SpeechModuleResolver): AudioInputPreflightCallback {
@@ -24,16 +24,16 @@ export function configureModuleSpeech(resolve: SpeechModuleResolver): AudioInput
await configureProvider(context.runtime, configuration.provider)
await configureActiveCardModules(context.runtime, {
speech: {
provider: configuration.provider.id,
model: configuration.provider.model,
provider: configuration.provider.id,
voice_id: configuration.voice,
},
})
await configureStorage(context.runtime, {
'settings/speech/active-provider': configuration.provider.id,
'settings/speech/active-model': configuration.provider.model,
'settings/speech/voice': configuration.voice,
'settings/speech/active-provider': configuration.provider.id,
'settings/speech/output-muted': String(configuration.muted ?? false),
'settings/speech/voice': configuration.voice,
})
}
}
packages/testing-audio/cases/shared/configurations/onboarding.ts
+1 -1
View File
@@ -8,7 +8,7 @@ export interface OnboardingConfiguration {
skipped?: boolean
}
type OnboardingResolver = (context: AudioInputPreflightContext) => OnboardingConfiguration | undefined | Promise<OnboardingConfiguration | undefined>
type OnboardingResolver = (context: AudioInputPreflightContext) => OnboardingConfiguration | Promise<OnboardingConfiguration | undefined> | undefined
/** Configures onboarding with values selected by one case. */
export function configureOnboarding(resolve: OnboardingResolver): AudioInputPreflightCallback {
packages/testing-audio/cases/shared/configurations/provider.ts
+2 -2
View File
@@ -17,9 +17,9 @@ export async function configureProvider(runtime: AudioInputSession, provider: Pr
credentials[configuredProvider.id] = configuredProvider.config
configured[configuredProvider.id] = {
id: configuredProvider.id,
definitionId: configuredProvider.definitionId,
config: configuredProvider.config,
definitionId: configuredProvider.definitionId,
id: configuredProvider.id,
status: 'configured',
}
added[configuredProvider.id] = true
packages/testing-audio/cases/shared/interactions/chat.ts
+5 -5
View File
@@ -9,6 +9,11 @@ export interface EnableChatMicrophoneOptions {
readiness?: 'streaming-transcription' | 'vad'
}
/** Returns the assistant messages on the current chat page. */
export function assistantMessages(runtime: AudioInputSession): Locator {
return runtime.page.locator('[data-chat-message-role="assistant"] .markdown-content')
}
/** Enables the chat microphone through the UI owned by the selected runtime. */
export async function enableChatMicrophone(
runtime: AudioInputSession,
@@ -81,11 +86,6 @@ export async function openChat(runtime: AudioInputSession): Promise<void> {
await runtime.page.locator('textarea').first().waitFor({ state: 'visible', timeout: 60_000 })
}
/** Returns the assistant messages on the current chat page. */
export function assistantMessages(runtime: AudioInputSession): Locator {
return runtime.page.locator('[data-chat-message-role="assistant"] .markdown-content')
}
async function openElectronChat(
app: NonNullable<AudioInputSession['electronApp']>,
chatButton: Locator,
packages/testing-audio/cases/shared/interactions/hearing-playground.ts
+12 -12
View File
@@ -4,6 +4,18 @@ import type { AudioInputSession } from '../../../src/types'
import { captureStreamingTranscription } from './streaming-transcription'
/** Enables microphone monitoring on an open hearing playground. */
export async function enableHearingPlaygroundMicrophone(page: Page): Promise<void> {
await captureStreamingTranscription(page, '[data-testid="hearing-playground-current"] p')
const modelBasedToggle = page.getByRole('switch').last()
if (await modelBasedToggle.isChecked())
await modelBasedToggle.click()
const monitorToggle = page.getByTestId('hearing-playground-monitor-toggle')
await monitorToggle.click()
}
/** Opens the Electron hearing playground for a case that selects this input UI. */
export async function openHearingPlayground(runtime: AudioInputSession): Promise<Page> {
const app = runtime.electronApp
@@ -26,18 +38,6 @@ export async function openHearingPlayground(runtime: AudioInputSession): Promise
return settingsPage
}
/** Enables microphone monitoring on an open hearing playground. */
export async function enableHearingPlaygroundMicrophone(page: Page): Promise<void> {
await captureStreamingTranscription(page, '[data-testid="hearing-playground-current"] p')
const modelBasedToggle = page.getByRole('switch').last()
if (await modelBasedToggle.isChecked())
await modelBasedToggle.click()
const monitorToggle = page.getByTestId('hearing-playground-monitor-toggle')
await monitorToggle.click()
}
/** Reads the requested number of final hearing playground transcripts. */
export async function readHearingPlaygroundTranscriptions(page: Page, count: number): Promise<string[]> {
const transcriptions = page.getByTestId('hearing-playground-transcript')
packages/testing-audio/cases/shared/providers/aliyun-nls.ts
+3 -3
View File
@@ -14,13 +14,13 @@ export function aliyunNlsAsr(options: AliyunNlsAsrOptions): ProviderConfiguratio
throw new Error('The Aliyun NLS Provider must be "aliyun-nls-transcription".')
return {
id: options.provider,
definitionId: options.provider,
model: 'aliyun-nls-v1',
config: {
accessKeyId: options.accessKeyId,
accessKeySecret: options.accessKeySecret,
appKey: options.appKey,
},
definitionId: options.provider,
id: options.provider,
model: 'aliyun-nls-v1',
}
}
packages/testing-audio/cases/shared/providers/apple-speech.ts
+3 -3
View File
@@ -9,11 +9,11 @@ export interface AppleSpeechAsrOptions {
/** Creates the macOS Apple Speech Provider configuration for one Electron case. */
export function appleSpeechAsr(options: AppleSpeechAsrOptions = {}): ProviderConfiguration {
return {
id: 'apple-speech-transcription',
definitionId: 'apple-speech-transcription',
model: 'apple-speech',
config: {
locale: options.locale ?? 'en-US',
},
definitionId: 'apple-speech-transcription',
id: 'apple-speech-transcription',
model: 'apple-speech',
}
}
packages/testing-audio/cases/shared/providers/openai.ts
+10 -10
View File
@@ -16,43 +16,43 @@ export interface OpenAISpeechProviderOptions extends OpenAIProviderOptions {
/** Creates an OpenAI-compatible ASR Provider configuration. */
export function openaiAsr(options: OpenAIProviderOptions): ProviderConfiguration {
return {
id: options.provider,
definitionId: options.provider,
model: options.model,
config: {
apiKey: options.apiKey,
baseUrl: options.baseUrl,
},
definitionId: options.provider,
id: options.provider,
model: options.model,
}
}
/** Creates an OpenAI-compatible LLM Provider configuration. */
export function openaiLlm(options: OpenAIProviderOptions): ProviderConfiguration {
return {
id: options.provider,
definitionId: options.provider,
model: options.model,
config: {
apiKey: options.apiKey,
baseUrl: options.baseUrl,
},
definitionId: options.provider,
id: options.provider,
model: options.model,
}
}
/** Creates an OpenAI-compatible TTS Provider configuration. */
export function openaiTts(options: OpenAISpeechProviderOptions): { provider: ProviderConfiguration, voice: string } {
return {
voice: options.voice,
provider: {
id: options.provider,
definitionId: options.provider,
model: options.model,
config: {
apiKey: options.apiKey,
baseUrl: options.baseUrl,
model: options.model,
voice: options.voice,
},
definitionId: options.provider,
id: options.provider,
model: options.model,
},
voice: options.voice,
}
}
packages/testing-audio/cases/single-utterance-pipeline/case.audio.test.ts
+2 -2
View File
@@ -20,13 +20,13 @@ describe('audio input pipeline', () => {
return undefined
return {
captureFormat: 'wav',
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
configureModuleConsciousness(async (context) => {
@@ -108,8 +108,8 @@ describe('audio input pipeline', () => {
return undefined
return {
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
captureFormat: 'pcm',
provider: aliyunNlsAsr({ accessKeyId, accessKeySecret, appKey, provider }),
}
}),
],
packages/testing-audio/cases/two-utterance-streaming/case.audio.test.ts
+2 -2
View File
@@ -19,13 +19,13 @@ describe('audio input pipeline', () => {
return undefined
return {
captureFormat: 'wav',
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
],
@@ -75,8 +75,8 @@ describe('audio input pipeline', () => {
return undefined
return {
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
captureFormat: 'pcm',
provider: aliyunNlsAsr({ accessKeyId, accessKeySecret, appKey, provider }),
}
}),
],
packages/testing-audio/src/describe.ts
+7 -7
View File
@@ -21,24 +21,19 @@ const audioTestAPI = createAudioTestAPI<
{ audio: AudioInputSession },
AudioInputPreflightContext
>({
preflight: definition => definition.preflight,
createPlans(name, testCase) {
const task = createAudioTestTask(name, testCase)
return [{
name: task.name,
definition: task,
metadata: {
input: fileURLToPath(task.input),
runtime: inject('fakemicRuntime').name,
},
name: task.name,
}]
},
async execute({ plan, task, invokeHandler, runPreflight }) {
async execute({ invokeHandler, plan, runPreflight, task }) {
await runAudioTestSession({
start() {
const microphoneInput = fileURLToPath(plan.definition.input)
return startFakemicRuntime<AudioInputSession>(microphoneInput)
},
async execute(session) {
await runPreflight({
env,
@@ -62,8 +57,13 @@ const audioTestAPI = createAudioTestAPI<
contentType: 'application/json',
})
},
start() {
const microphoneInput = fileURLToPath(plan.definition.input)
return startFakemicRuntime<AudioInputSession>(microphoneInput)
},
})
},
preflight: definition => definition.preflight,
})
/** Groups AIRI audio-input tests in the Vitest task tree. */
packages/testing-audio/src/expect-extend.test.ts
+12 -12
View File
@@ -28,16 +28,26 @@ describe('audio input matchers', () => {
})
})
function actionEvent(storeId: string, actionName: string): PiniaActionEvent {
return {
actionName,
invocationId: `${storeId}:${actionName}`,
status: 'completed',
storeId,
timestamp: 0,
}
}
function createAudioInputSession(
transcriptions: string[],
actions: PiniaActionEvent[] = [],
): AudioInputObservations {
return {
capturedTranscriptionAudio: async () => [],
streamingTranscriptionUpdates: async () => [],
transcriptionResults: async () => transcriptions,
completedSpans: async () => [],
piniaActionEvents: async () => actions,
streamingTranscriptionUpdates: async () => [],
transcriptionResults: async () => transcriptions,
waitForPiniaAction: async () => {
throw new Error('This matcher fixture does not observe Pinia actions.')
},
@@ -48,13 +58,3 @@ function createAudioInputSession(
waitForVadReady: async () => {},
}
}
function actionEvent(storeId: string, actionName: string): PiniaActionEvent {
return {
actionName,
invocationId: `${storeId}:${actionName}`,
status: 'completed',
storeId,
timestamp: 0,
}
}
packages/testing-audio/src/expect-extend.ts
+54 -54
View File
@@ -7,51 +7,37 @@ export interface CapturedTranscriptionAudioExpectation {
minimumBytes: number
}
export interface TranscriptionExpectationOptions {
/** @default 'exact' */
match?: 'exact' | 'contains'
}
/** Sets the wait limit for a transcription action assertion. */
export interface TranscriptionActionExpectationOptions {
/** Maximum time to wait for a completed ASR action. @default 60000 */
timeout?: number
}
export interface TranscriptionExpectationOptions {
/** @default 'exact' */
match?: 'contains' | 'exact'
}
declare module 'vitest' {
interface Assertion<T> {
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
? (expected: CapturedTranscriptionAudioExpectation) => Promise<void>
: never
toHaveCompletedTranscription: T extends AudioInputObservations
? (options?: TranscriptionActionExpectationOptions) => Promise<void>
: never
toHaveTranscriptions: T extends AudioInputObservations
? (
expected: ReadonlyArray<ReadonlyArray<string>>,
options?: TranscriptionExpectationOptions,
) => Promise<void>
: never
toHaveCompletedTranscription: T extends AudioInputObservations
? (options?: TranscriptionActionExpectationOptions) => Promise<void>
: never
}
}
/** Vitest expect with AIRI audio-input matcher types. */
export const expect = vitestExpect
/**
* Normalizes speech text for transcript comparison.
*
* @example
* normalizeSpeechText(' Hello, AIRI! ')
* // => 'helloairi'
*/
function normalizeSpeechText(value: string): string {
return value
.normalize('NFKC')
.toLocaleLowerCase()
.replace(/[^\p{L}\p{N}]+/gu, '')
}
/** Installs asynchronous matchers for AIRI audio-input observations. */
export function installAudioInputMatchers(): void {
vitestExpect.extend({
@@ -62,8 +48,8 @@ export function installAudioInputMatchers(): void {
const format = session.transcriptionCaptureFormat
if (!format) {
return {
pass: false,
message: () => 'The active transcription Provider does not expose uploaded audio.',
pass: false,
}
}
@@ -76,10 +62,22 @@ export function installAudioInputMatchers(): void {
const pass = captures.length === expected.count && !invalidCapture
return {
pass,
message: () => pass
? 'Expected the session not to contain valid transcription audio.'
: `Expected ${expected.count} ${format} capture(s) with at least ${expected.minimumBytes} bytes.`,
pass,
}
},
async toHaveCompletedTranscription(
session: AudioInputObservations,
options: TranscriptionActionExpectationOptions = {},
) {
const result = await waitForTranscription(session, options.timeout ?? 60_000)
return {
message: () => result.complete
? 'Expected the transcription action not to complete.'
: `Expected the transcription action to complete, but ${result.summary}.`,
pass: result.complete,
}
},
async toHaveTranscriptions(
@@ -99,46 +97,34 @@ export function installAudioInputMatchers(): void {
))
return {
pass,
message: () => pass
? 'Expected the session not to contain the specified transcriptions.'
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
}
},
async toHaveCompletedTranscription(
session: AudioInputObservations,
options: TranscriptionActionExpectationOptions = {},
) {
const result = await waitForTranscription(session, options.timeout ?? 60_000)
return {
pass: result.complete,
message: () => result.complete
? 'Expected the transcription action not to complete.'
: `Expected the transcription action to complete, but ${result.summary}.`,
pass,
}
},
})
}
const transcriptionActions = [
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForRecording' },
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForMediaStream' },
]
async function waitForTranscription(
session: AudioInputObservations,
timeout: number,
): Promise<{ complete: boolean, failed: boolean, summary: string }> {
const deadline = Date.now() + timeout
let result = transcriptionResult(await session.piniaActionEvents())
while (!result.complete && !result.failed && Date.now() < deadline) {
await new Promise(resolve => setTimeout(resolve, 100))
result = transcriptionResult(await session.piniaActionEvents())
}
return result
/**
* Normalizes speech text for transcript comparison.
*
* @example
* normalizeSpeechText(' Hello, AIRI! ')
* // => 'helloairi'
*/
function normalizeSpeechText(value: string): string {
return value
.normalize('NFKC')
.toLocaleLowerCase()
.replace(/[^\p{L}\p{N}]+/gu, '')
}
const transcriptionActions = [
{ actionName: 'transcribeForRecording', storeId: 'modules:hearing:speech:audio-input-pipeline' },
{ actionName: 'transcribeForMediaStream', storeId: 'modules:hearing:speech:audio-input-pipeline' },
]
function transcriptionResult(
events: Awaited<ReturnType<AudioInputObservations['piniaActionEvents']>>,
): { complete: boolean, failed: boolean, summary: string } {
@@ -159,3 +145,17 @@ function transcriptionResult(
summary: 'ASR did not complete',
}
}
async function waitForTranscription(
session: AudioInputObservations,
timeout: number,
): Promise<{ complete: boolean, failed: boolean, summary: string }> {
const deadline = Date.now() + timeout
let result = transcriptionResult(await session.piniaActionEvents())
while (!result.complete && !result.failed && Date.now() < deadline) {
await new Promise(resolve => setTimeout(resolve, 100))
result = transcriptionResult(await session.piniaActionEvents())
}
return result
}
packages/testing-audio/src/runtimes/prepare-electron.ts
+1 -1
View File
@@ -19,10 +19,10 @@ export default async function prepareElectronRuntime(context: FakemicElectronPre
await page.evaluate(stubForBrowser, piniaActionTracingChannelName)
return createSession({
close: context.close,
electronApp: context.app,
page,
target: 'electron',
close: context.close,
})
}
packages/testing-audio/src/runtimes/prepare-web.ts
+1 -1
View File
@@ -15,8 +15,8 @@ export default async function prepareWebRuntime(context: FakemicWebPrepareContex
await page.goto(context.runtime.url)
return createSession({
close: context.close,
page,
target: 'web',
close: context.close,
})
}
packages/testing-audio/src/setup/browser-probe.ts
+17 -17
View File
@@ -1,6 +1,12 @@
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
/** Returns the completed browser spans that match the optional span name. */
export function readCompletedSpans(name?: string) {
const spans = window.__airiAudioInputE2E?.spans ?? []
return name ? spans.filter(span => span.name === name) : spans
}
/**
* Installs passive browser probes before the application starts.
*
@@ -134,7 +140,16 @@ export function stubForBrowser(piniaActionChannelName: string) {
}
}
send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
close(code?: number, reason?: string): void {
const audioChunks = audioChunksBySocket.get(this)
if (capturesAliyunNlsBySocket.get(this) && audioChunks && !capturedSockets.has(this) && audioChunks.length) {
capturePcmAudio(this)
}
super.close(code, reason)
}
send(data: ArrayBufferLike | ArrayBufferView | Blob | string): void {
const audioChunks = audioChunksBySocket.get(this)
if (capturesAliyunNlsBySocket.get(this) && audioChunks && typeof data !== 'string') {
if (data instanceof Blob) {
@@ -151,7 +166,7 @@ export function stubForBrowser(piniaActionChannelName: string) {
capturePcmAudio(this)
}
let outboundData: string | Blob | BufferSource
let outboundData: Blob | BufferSource | string
if (typeof data === 'string' || data instanceof Blob || data instanceof ArrayBuffer) {
outboundData = data
}
@@ -166,15 +181,6 @@ export function stubForBrowser(piniaActionChannelName: string) {
super.send(outboundData)
}
close(code?: number, reason?: string): void {
const audioChunks = audioChunksBySocket.get(this)
if (capturesAliyunNlsBySocket.get(this) && audioChunks && !capturedSockets.has(this) && audioChunks.length) {
capturePcmAudio(this)
}
super.close(code, reason)
}
}
window.WebSocket = CaptureAliyunNlsWebSocket
@@ -203,9 +209,3 @@ export function stubForBrowser(piniaActionChannelName: string) {
channel.addEventListener('message', captureSpan)
}
/** Returns the completed browser spans that match the optional span name. */
export function readCompletedSpans(name?: string) {
const spans = window.__airiAudioInputE2E?.spans ?? []
return name ? spans.filter(span => span.name === name) : spans
}
packages/testing-audio/src/setup/session.ts
+58 -58
View File
@@ -13,18 +13,13 @@ import { readCompletedSpans } from './browser-probe'
/** Creates the runtime session for one audio test. */
export function createSession(options: {
close: () => Promise<void>
electronApp?: ElectronApplication
page: Page
target: AudioInputTarget
close: () => Promise<void>
transcriptionCaptureFormat?: AudioCapture['format']
}): AudioInputSession {
const session: AudioInputSession = {
electronApp: options.electronApp,
page: options.page,
runtimePage: options.page,
target: options.target,
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
activatePage(page) {
session.page = page
},
@@ -56,17 +51,11 @@ export function createSession(options: {
}
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
return capturedAudio.map(audio => ({
format: audio.format,
data: Buffer.from(audio.base64, 'base64'),
format: audio.format,
}))
},
streamingTranscriptionUpdates: () => session.page.evaluate(() => window.__airiAudioInputE2E?.streamingTranscriptionUpdates ?? []),
async transcriptionResults(count) {
await options.page.waitForFunction(expectedCount => (
(window.__airiAudioInputE2E?.transcriptionResults.length ?? 0) >= expectedCount
), count, { timeout: 60_000 })
return options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionResults ?? [])
},
close: options.close,
async completedSpans(name) {
const runtimeSpans = await options.page.evaluate(readCompletedSpans, name)
if (session.page === options.page)
@@ -75,28 +64,30 @@ export function createSession(options: {
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
return [...runtimeSpans, ...interactionSpans]
},
async waitForTurn(waitOptions = {}) {
const timeout = waitOptions.timeout ?? 60_000
const deadline = Date.now() + timeout
let turn = createTurnObservation(await session.completedSpans())
while (!turn && Date.now() < deadline) {
await new Promise(resolve => setTimeout(resolve, 100))
turn = createTurnObservation(await session.completedSpans())
}
if (!turn)
throw new Error('Timed out waiting for a completed LLM and speech turn.')
turn.chat.messages = await readChatMessages(session.page)
return turn
},
electronApp: options.electronApp,
page: options.page,
piniaActionEvents: () => options.page.evaluate(() => window.__airiAudioInputE2E?.piniaActionEvents ?? []),
async waitForVadReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
runtimePage: options.page,
async snapshot() {
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
const interactionState = session.page === options.page
? runtimeState
: await session.page.evaluate(() => window.__airiAudioInputE2E)
return {
piniaActionEvents: runtimeState?.piniaActionEvents ?? [],
spans: runtimeState?.spans ?? [],
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
transcriptionResults: runtimeState?.transcriptionResults ?? [],
}
},
async waitForStreamingTranscriptionReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
streamingTranscriptionUpdates: () => session.page.evaluate(() => window.__airiAudioInputE2E?.streamingTranscriptionUpdates ?? []),
target: options.target,
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
async transcriptionResults(count) {
await options.page.waitForFunction(expectedCount => (
(window.__airiAudioInputE2E?.transcriptionResults.length ?? 0) >= expectedCount
), count, { timeout: 60_000 })
return options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionResults ?? [])
},
async waitForPiniaAction(waitOptions) {
return options.page.evaluate(({ actionName, channelName, status, storeId, timeout }) => new Promise<PiniaActionEvent>((resolve, reject) => {
@@ -123,19 +114,28 @@ export function createSession(options: {
timeout: waitOptions.timeout ?? 60_000,
})
},
async snapshot() {
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
const interactionState = session.page === options.page
? runtimeState
: await session.page.evaluate(() => window.__airiAudioInputE2E)
return {
piniaActionEvents: runtimeState?.piniaActionEvents ?? [],
spans: runtimeState?.spans ?? [],
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
transcriptionResults: runtimeState?.transcriptionResults ?? [],
async waitForStreamingTranscriptionReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
},
async waitForTurn(waitOptions = {}) {
const timeout = waitOptions.timeout ?? 60_000
const deadline = Date.now() + timeout
let turn = createTurnObservation(await session.completedSpans())
while (!turn && Date.now() < deadline) {
await new Promise(resolve => setTimeout(resolve, 100))
turn = createTurnObservation(await session.completedSpans())
}
if (!turn)
throw new Error('Timed out waiting for a completed LLM and speech turn.')
turn.chat.messages = await readChatMessages(session.page)
return turn
},
async waitForVadReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
},
close: options.close,
}
return session
@@ -168,8 +168,8 @@ function createTurnObservation(spans: Awaited<ReturnType<AudioInputSession['comp
}))
return {
id: turnId,
chat: { messages: [] },
id: turnId,
llm: {
inputMessages: inputMessageRoles.map(role => ({ role })),
outputCharacters: numberAttribute(llmSpan, IOAttributes.LLMTextLength),
@@ -179,6 +179,16 @@ function createTurnObservation(spans: Awaited<ReturnType<AudioInputSession['comp
}
}
function numberArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number[] {
const value = span.attributes[name]
return Array.isArray(value) ? value.filter(item => typeof item === 'number') : []
}
function numberAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number {
const value = span.attributes[name]
return typeof value === 'number' ? value : 0
}
async function readChatMessages(page: Page): Promise<AudioInputChatMessage[]> {
return page.locator('[data-chat-message-role]').evaluateAll(elements => elements.flatMap((element) => {
const role = element.getAttribute('data-chat-message-role')
@@ -189,22 +199,12 @@ async function readChatMessages(page: Page): Promise<AudioInputChatMessage[]> {
}))
}
function numberAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number {
function stringArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string[] {
const value = span.attributes[name]
return typeof value === 'number' ? value : 0
}
function numberArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number[] {
const value = span.attributes[name]
return Array.isArray(value) ? value.filter(item => typeof item === 'number') : []
return Array.isArray(value) ? value.filter(item => typeof item === 'string') : []
}
function stringAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string {
const value = span.attributes[name]
return typeof value === 'string' ? value : ''
}
function stringArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string[] {
const value = span.attributes[name]
return Array.isArray(value) ? value.filter(item => typeof item === 'string') : []
}
packages/testing-audio/src/types.ts
+65 -65
View File
@@ -5,32 +5,6 @@ import type { ElectronApplication, Page } from 'playwright'
export const audioInputTargets = ['web', 'electron'] as const
export type AudioInputTarget = (typeof audioInputTargets)[number]
/** Values available when one case resolves its preflight callbacks. */
export interface AudioInputPreflightContext {
/** Environment variables loaded for this case. */
env: Readonly<NodeJS.ProcessEnv>
/** Clean runtime that the configuration callback can prepare. */
runtime: AudioInputSession
/** Skips this case when its environment does not satisfy a case constraint. */
skip: (condition: unknown, note?: string) => void
}
/** One optional case callback that resolves configuration from its environment. */
export type AudioInputPreflightCallback = AudioTestPreflightCallback<AudioInputPreflightContext>
/** One AIRI audio-input test definition. */
export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
/** Snapshot of the observable AIRI audio pipeline state. */
export interface AudioInputSnapshot {
piniaActionEvents: PiniaActionEvent[]
spans: SerializedIOSpan[]
streamingTranscriptionUpdates: string[]
transcriptionResults: string[]
}
/** One rendered chat message from the completed turn. */
export interface AudioInputChatMessage {
role: 'assistant' | 'user'
@@ -47,47 +21,16 @@ export interface AudioInputLLMOutputChunk {
characters: number
}
/** One synthesized audio segment routed from the LLM turn. */
export interface AudioInputTTSSegment {
durationMs: number
text: string
}
/** Structured observations for one completed voice interaction turn. */
export interface AudioInputTurn {
id: string
chat: {
messages: AudioInputChatMessage[]
}
llm: {
inputMessages: AudioInputLLMMessage[]
outputCharacters: number
outputChunks: AudioInputLLMOutputChunk[]
}
tts: {
audioSegments: AudioInputTTSSegment[]
}
}
/** Observable audio values used by AIRI matchers. */
export interface AudioInputObservations {
/** Capture format used by the active transcription Provider. */
transcriptionCaptureFormat?: AudioCaptureFormat
capturedTranscriptionAudio: (count: number) => Promise<AudioCapture[]>
streamingTranscriptionUpdates: () => Promise<string[]>
transcriptionResults: (count: number) => Promise<string[]>
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
/** Waits until all speech work for the latest LLM turn is complete. */
waitForTurn: (options?: {
/** @default 60000 */
timeout?: number
}) => Promise<AudioInputTurn>
/** Returns the Pinia action events collected by the runtime probe. */
piniaActionEvents: () => Promise<PiniaActionEvent[]>
/** Waits until the VAD audio graph is connected to the microphone stream. */
waitForVadReady: () => Promise<void>
/** Waits until a streaming transcription transport accepts microphone audio. */
waitForStreamingTranscriptionReady: () => Promise<void>
streamingTranscriptionUpdates: () => Promise<string[]>
/** Capture format used by the active transcription Provider. */
transcriptionCaptureFormat?: AudioCaptureFormat
transcriptionResults: (count: number) => Promise<string[]>
/**
* Waits for the next matching Pinia action event after this method is called.
*
@@ -100,26 +43,83 @@ export interface AudioInputObservations {
* await completed
*/
waitForPiniaAction: (options: {
storeId: string
actionName: string
/** @default 'completed' */
status?: PiniaActionEventStatus
storeId: string
/** @default 60000 */
timeout?: number
}) => Promise<PiniaActionEvent>
/** Waits until a streaming transcription transport accepts microphone audio. */
waitForStreamingTranscriptionReady: () => Promise<void>
/** Waits until all speech work for the latest LLM turn is complete. */
waitForTurn: (options?: {
/** @default 60000 */
timeout?: number
}) => Promise<AudioInputTurn>
/** Waits until the VAD audio graph is connected to the microphone stream. */
waitForVadReady: () => Promise<void>
}
/** One optional case callback that resolves configuration from its environment. */
export type AudioInputPreflightCallback = AudioTestPreflightCallback<AudioInputPreflightContext>
/** Values available when one case resolves its preflight callbacks. */
export interface AudioInputPreflightContext {
/** Environment variables loaded for this case. */
env: Readonly<NodeJS.ProcessEnv>
/** Clean runtime that the configuration callback can prepare. */
runtime: AudioInputSession
/** Skips this case when its environment does not satisfy a case constraint. */
skip: (condition: unknown, note?: string) => void
}
/** Runtime handle for one AIRI audio-input test. */
export interface AudioInputSession extends AudioInputObservations, AudioTestSession {
/** Selects the page used by subsequent case interactions. */
activatePage: (page: Page) => void
/** Electron application for Electron tasks. */
electronApp?: ElectronApplication
/** Page used by case interactions. */
page: Page
/** Page that owns the audio-input pipeline. */
runtimePage: Page
snapshot: () => Promise<AudioInputSnapshot>
/** Runtime selected for this concrete task. */
target: AudioInputTarget
/** Selects the page used by subsequent case interactions. */
activatePage: (page: Page) => void
snapshot: () => Promise<AudioInputSnapshot>
}
/** Snapshot of the observable AIRI audio pipeline state. */
export interface AudioInputSnapshot {
piniaActionEvents: PiniaActionEvent[]
spans: SerializedIOSpan[]
streamingTranscriptionUpdates: string[]
transcriptionResults: string[]
}
export type AudioInputTarget = (typeof audioInputTargets)[number]
/** One AIRI audio-input test definition. */
export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
/** One synthesized audio segment routed from the LLM turn. */
export interface AudioInputTTSSegment {
durationMs: number
text: string
}
/** Structured observations for one completed voice interaction turn. */
export interface AudioInputTurn {
chat: {
messages: AudioInputChatMessage[]
}
id: string
llm: {
inputMessages: AudioInputLLMMessage[]
outputCharacters: number
outputChunks: AudioInputLLMOutputChunk[]
}
tts: {
audioSegments: AudioInputTTSSegment[]
}
}
packages/testing-audio/src/types/browser.d.ts
Vendored
+4 -4
View File
@@ -12,10 +12,6 @@ declare global {
vadReady: boolean
}
interface Window {
__airiAudioInputE2E?: BrowserAudioInputState
}
// NOTICE:
// TypeScript 5.9 does not declare the Baseline 2025 Uint8Array Base64 methods.
// The test runs in current Playwright and Electron Chromium runtimes that implement this API.
@@ -27,6 +23,10 @@ declare global {
omitPadding?: boolean
}) => string
}
interface Window {
__airiAudioInputE2E?: BrowserAudioInputState
}
}
export {}
packages/testing-audio/vitest.config.ts
+8 -8
View File
@@ -8,28 +8,28 @@ export default defineConfig({
test: {
projects: [
fakemic({
name: 'audio-web',
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.web.test.ts'],
name: 'audio-web',
runtime: web({
context: { permissions: ['microphone'] },
name: 'web',
prepare: new URL('./src/runtimes/prepare-web.ts', import.meta.url).href,
url: 'http://127.0.0.1:4173/',
context: { permissions: ['microphone'] },
preview: {
configFile: join(import.meta.dirname, '../../apps/stage-web/vite.config.ts'),
root: join(import.meta.dirname, '../../apps/stage-web'),
},
url: 'http://127.0.0.1:4173/',
}),
}),
fakemic({
name: 'audio-electron',
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.electron.test.ts'],
name: 'audio-electron',
runtime: electron({
name: 'electron',
prepare: new URL('./src/runtimes/prepare-electron.ts', import.meta.url).href,
entry: join(import.meta.dirname, '../../apps/stage-tamagotchi/out/main/index.js'),
args: ['--no-sandbox'],
cwd: join(import.meta.dirname, '../..'),
entry: join(import.meta.dirname, '../../apps/stage-tamagotchi/out/main/index.js'),
name: 'electron',
prepare: new URL('./src/runtimes/prepare-electron.ts', import.meta.url).href,
temporaryUserData: {
env: 'APP_USER_DATA_PATH',
prefix: 'airi-testing-audio-',
@@ -39,8 +39,8 @@ export default defineConfig({
{
extends: true,
test: {
name: 'unit',
include: ['src/**/*.test.ts'],
name: 'unit',
},
},
],