feat(testing-audio): now entire audio input/output pipeline can be tested
This commit is contained in:
Neko Ayaka
@@ -0,0 +1 @@
|
||||
test-results/
|
||||
@@ -0,0 +1,127 @@
|
||||
# Testing audio
|
||||
|
||||
This package runs recorded microphone tests through the real AIRI audio pipeline.
|
||||
|
||||
```text
|
||||
input.wav -> virtual microphone -> VAD -> ASR -> LLM -> TTS -> playback -> UI
|
||||
```
|
||||
|
||||
The package exports audio-aware `describe`, `it`, and `expect` APIs. Vitest owns the task tree and result protocol. Each project starts its configured Playwright runtime.
|
||||
|
||||
`src` only owns test scheduling, runtime startup, browser probes, and matchers. Case-specific environment, storage, Provider, route, and UI operations live under `cases/shared`:
|
||||
|
||||
```text
|
||||
cases/shared/
|
||||
configurations/ # preflight callbacks that configure one concern
|
||||
interactions/ # reusable UI operations selected by a case
|
||||
```
|
||||
|
||||
## Configure a case
|
||||
|
||||
The optional `preflight` field is an ordered callback array. The fake-microphone Vitest integration starts the runtime before it invokes these callbacks. Each callback receives:
|
||||
|
||||
- `env`: the process environment for this case
|
||||
- `runtime`: the clean Playwright runtime
|
||||
- `skip`: Vitest's case-level skip control
|
||||
|
||||
Every case explicitly selects the configuration it needs. Do not create one shared “complete pipeline” preflight that hides the Provider combination.
|
||||
|
||||
```ts
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { enableChatMicrophone } from '../shared/interactions'
|
||||
import { openaiAsr } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
it('transcribes a greeting', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
])
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
A case can independently choose its VAD behavior, ASR, LLM, and TTS configuration. A configuration callback can read its own environment, write local storage, use another persistence mechanism, or deliberately leave onboarding incomplete.
|
||||
|
||||
## Provider environment
|
||||
|
||||
Each case selects its environment variables. `loadCaseEnvironment` uses Vite test mode to read repository, `packages/stage-ui`, and package environment files. Process variables have the highest priority.
|
||||
|
||||
Put local test credentials in `packages/testing-audio/.env.test.local`. Git ignores this file.
|
||||
|
||||
The OpenAI-compatible Provider helpers do not read the environment. Pass the endpoint, API key, model, and voice from the case callback.
|
||||
|
||||
The included cases use these explicit variables:
|
||||
|
||||
```dotenv
|
||||
TESTING_AUDIO_ASR_PROVIDER=openai-compatible-audio-transcription
|
||||
TESTING_AUDIO_ASR_MODEL=whisper-1
|
||||
TESTING_AUDIO_ASR_API_BASE_URL=https://api.openai.com/v1/
|
||||
TESTING_AUDIO_ASR_API_KEY=...
|
||||
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER=aliyun-nls-transcription
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID=...
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET=...
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY=...
|
||||
|
||||
TESTING_AUDIO_LLM_PROVIDER=openai-compatible
|
||||
TESTING_AUDIO_LLM_MODEL=gpt-4o-mini
|
||||
TESTING_AUDIO_LLM_API_BASE_URL=https://api.openai.com/v1/
|
||||
TESTING_AUDIO_LLM_API_KEY=...
|
||||
|
||||
TESTING_AUDIO_TTS_PROVIDER=openai-compatible-audio-speech
|
||||
TESTING_AUDIO_TTS_MODEL=tts-1
|
||||
TESTING_AUDIO_TTS_VOICE=alloy
|
||||
TESTING_AUDIO_TTS_API_BASE_URL=https://api.openai.com/v1/
|
||||
TESTING_AUDIO_TTS_API_KEY=...
|
||||
```
|
||||
|
||||
These tests send audio and text to external Providers. Each run can incur Provider charges.
|
||||
|
||||
## Run the tests
|
||||
|
||||
Build both targets and run all runtime projects:
|
||||
|
||||
```bash
|
||||
pnpm -F @proj-airi/testing-audio test:run
|
||||
```
|
||||
|
||||
Use existing builds:
|
||||
|
||||
```bash
|
||||
pnpm -F @proj-airi/testing-audio test:existing-builds
|
||||
```
|
||||
|
||||
Run one runtime project:
|
||||
|
||||
```bash
|
||||
pnpm -F @proj-airi/testing-audio exec vitest run --project audio-web
|
||||
pnpm -F @proj-airi/testing-audio exec vitest run --project audio-electron
|
||||
```
|
||||
|
||||
Use `*.audio.web.test.ts` for Web-only cases. Use `*.audio.electron.test.ts` for Electron-only cases. The `*.audio.test.ts` pattern runs in both projects.
|
||||
|
||||
Do not use Vitest Browser Mode for these cases. Each task needs a case-specific Chromium fake-microphone process argument.
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { enableChatMicrophone } from '../shared/interactions'
|
||||
import { aliyunNlsAsr, openaiAsr } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
// The fixture uses the repository recording at docs/content/en/blog/DevLog-2025.03.20/assets/ashley-pitch-test.mp3.
|
||||
// It has 20 seconds of leading silence and 4 seconds of trailing silence in mono 16 kHz PCM WAV format.
|
||||
it('does not preserve the complete phrase after long leading silence', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
// This regression isolates AIRI's default VAD and one explicit ASR Provider.
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
|
||||
if (audio.transcriptionCaptureFormat) {
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
}
|
||||
|
||||
// The VAD upload currently drops part of the sentence before it sends the recording to ASR.
|
||||
await expect(audio).not.toHaveTranscriptions([
|
||||
['There is no meaning to your existence, just let go.'],
|
||||
])
|
||||
})
|
||||
|
||||
it('does not preserve the complete phrase with Aliyun NLS', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
|
||||
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
|
||||
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
|
||||
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
|
||||
context.skip(
|
||||
!provider || !accessKeyId || !accessKeySecret || !appKey,
|
||||
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
|
||||
)
|
||||
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
|
||||
captureFormat: 'pcm',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
|
||||
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
await expect(audio).not.toHaveTranscriptions([
|
||||
['There is no meaning to your existence, just let go.'],
|
||||
])
|
||||
})
|
||||
})
|
||||
Binary file not shown.
@@ -0,0 +1,35 @@
|
||||
import type { AiriCard, AiriExtension } from '@proj-airi/stage-ui/types'
|
||||
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
type ActiveCardModules = Partial<Pick<AiriExtension['modules'], 'consciousness' | 'speech'>>
|
||||
|
||||
/** Updates Provider selections that the active AIRI Card reapplies during application startup. */
|
||||
export async function configureActiveCardModules(
|
||||
runtime: AudioInputSession,
|
||||
modules: ActiveCardModules,
|
||||
): Promise<void> {
|
||||
await runtime.runtimePage.evaluate(({ configuredModules }) => {
|
||||
const serializedCards = localStorage.getItem('airi-cards')
|
||||
if (!serializedCards)
|
||||
throw new Error('The AIRI Card store is not initialized.')
|
||||
|
||||
const activeCardId = localStorage.getItem('airi-card-active-id') ?? 'default'
|
||||
const cards = JSON.parse(serializedCards) as Array<[string, AiriCard]>
|
||||
const activeCard = cards.find(([cardId]) => cardId === activeCardId)?.[1]
|
||||
if (!activeCard)
|
||||
throw new Error(`The active AIRI Card "${activeCardId}" does not exist.`)
|
||||
|
||||
const currentModules = activeCard.extensions.airi.modules
|
||||
activeCard.extensions.airi.modules = {
|
||||
...currentModules,
|
||||
...(configuredModules.consciousness
|
||||
? { consciousness: configuredModules.consciousness }
|
||||
: {}),
|
||||
...(configuredModules.speech
|
||||
? { speech: { ...currentModules.speech, ...configuredModules.speech } }
|
||||
: {}),
|
||||
}
|
||||
localStorage.setItem('airi-cards', JSON.stringify(cards))
|
||||
}, { configuredModules: modules })
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { AudioInputPreflightContext } from '../../../src/types'
|
||||
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
import { findWorkspaceDir } from '@pnpm/find-workspace-dir'
|
||||
import { loadEnv } from 'vite'
|
||||
|
||||
/** Loads Vite test-mode environment files and applies the environment of the current case. */
|
||||
export async function loadCaseEnvironment(
|
||||
environment: AudioInputPreflightContext['env'],
|
||||
): Promise<Record<string, string | undefined>> {
|
||||
const repositoryRoot = await findWorkspaceDir(import.meta.dirname)
|
||||
if (!repositoryRoot)
|
||||
throw new Error(`Unable to find the pnpm workspace from ${import.meta.dirname}`)
|
||||
|
||||
const repositoryEnvironment = loadEnv('test', repositoryRoot, '')
|
||||
// Shared Provider development variables live in stage-ui. These values override repository files.
|
||||
const stageUiEnvironment = loadEnv('test', resolve(repositoryRoot, 'packages/stage-ui'), '')
|
||||
// Audio case credentials belong to this package. These values override shared Provider variables.
|
||||
const testingAudioEnvironment = loadEnv('test', resolve(repositoryRoot, 'packages/testing-audio'), '')
|
||||
// The case process has the highest priority so that CI and shell values override local files.
|
||||
return { ...repositoryEnvironment, ...stageUiEnvironment, ...testingAudioEnvironment, ...environment }
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
export { loadCaseEnvironment } from './environment'
|
||||
export { configureModuleConsciousness } from './module-consciousness'
|
||||
export type { ConsciousnessModuleConfiguration } from './module-consciousness'
|
||||
export { configureModuleHearing } from './module-hearing'
|
||||
export type { HearingModuleConfiguration } from './module-hearing'
|
||||
export { configureModuleSpeech } from './module-speech'
|
||||
export type { SpeechModuleConfiguration } from './module-speech'
|
||||
export { configureOnboarding } from './onboarding'
|
||||
export type { OnboardingConfiguration } from './onboarding'
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
import type { ProviderConfiguration } from './provider'
|
||||
|
||||
import { configureActiveCardModules } from './active-card'
|
||||
import { configureProvider } from './provider'
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface ConsciousnessModuleConfiguration {
|
||||
provider: ProviderConfiguration
|
||||
}
|
||||
|
||||
type ConsciousnessModuleResolver = (context: AudioInputPreflightContext) => ConsciousnessModuleConfiguration | undefined | Promise<ConsciousnessModuleConfiguration | undefined>
|
||||
|
||||
/** Configures the consciousness module with the LLM Provider selected by one case. */
|
||||
export function configureModuleConsciousness(resolve: ConsciousnessModuleResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureProvider(context.runtime, configuration.provider)
|
||||
await configureActiveCardModules(context.runtime, {
|
||||
consciousness: {
|
||||
provider: configuration.provider.id,
|
||||
model: configuration.provider.model,
|
||||
},
|
||||
})
|
||||
await configureStorage(context.runtime, {
|
||||
'settings/consciousness/active-provider': configuration.provider.id,
|
||||
'settings/consciousness/active-model': configuration.provider.model,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { AudioCaptureFormat } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
import type { ProviderConfiguration } from './provider'
|
||||
|
||||
import { configureProvider } from './provider'
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface HearingModuleConfiguration {
|
||||
/** @default undefined */
|
||||
captureFormat?: AudioCaptureFormat
|
||||
/** @default false */
|
||||
microphoneEnabled?: boolean
|
||||
provider: ProviderConfiguration
|
||||
}
|
||||
|
||||
type HearingModuleResolver = (context: AudioInputPreflightContext) => HearingModuleConfiguration | undefined | Promise<HearingModuleConfiguration | undefined>
|
||||
|
||||
/** Configures the hearing module with the ASR Provider selected by one case. */
|
||||
export function configureModuleHearing(resolve: HearingModuleResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureProvider(context.runtime, configuration.provider)
|
||||
const settings: Record<string, string> = {
|
||||
'settings/hearing/active-provider': configuration.provider.id,
|
||||
'settings/hearing/active-model': configuration.provider.model,
|
||||
'settings/audio/input/enabled': String(configuration.microphoneEnabled ?? false),
|
||||
}
|
||||
|
||||
if (context.runtime.target === 'electron') {
|
||||
const microphoneInput = await context.runtime.runtimePage.evaluate(async () => {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices()
|
||||
return devices.find(device => device.kind === 'audioinput' && device.label.includes('Fake'))?.deviceId
|
||||
})
|
||||
if (!microphoneInput)
|
||||
throw new Error('Chromium did not expose the file-backed fake microphone.')
|
||||
settings['settings/audio/input'] = microphoneInput
|
||||
}
|
||||
|
||||
await configureStorage(context.runtime, settings)
|
||||
context.runtime.transcriptionCaptureFormat = configuration.captureFormat
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
import type { ProviderConfiguration } from './provider'
|
||||
|
||||
import { configureActiveCardModules } from './active-card'
|
||||
import { configureProvider } from './provider'
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface SpeechModuleConfiguration {
|
||||
/** @default false */
|
||||
muted?: boolean
|
||||
provider: ProviderConfiguration
|
||||
voice: string
|
||||
}
|
||||
|
||||
type SpeechModuleResolver = (context: AudioInputPreflightContext) => SpeechModuleConfiguration | undefined | Promise<SpeechModuleConfiguration | undefined>
|
||||
|
||||
/** Configures the speech module with the TTS Provider selected by one case. */
|
||||
export function configureModuleSpeech(resolve: SpeechModuleResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureProvider(context.runtime, configuration.provider)
|
||||
await configureActiveCardModules(context.runtime, {
|
||||
speech: {
|
||||
provider: configuration.provider.id,
|
||||
model: configuration.provider.model,
|
||||
voice_id: configuration.voice,
|
||||
},
|
||||
})
|
||||
await configureStorage(context.runtime, {
|
||||
'settings/speech/active-provider': configuration.provider.id,
|
||||
'settings/speech/active-model': configuration.provider.model,
|
||||
'settings/speech/voice': configuration.voice,
|
||||
'settings/speech/output-muted': String(configuration.muted ?? false),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface OnboardingConfiguration {
|
||||
completed: boolean
|
||||
/** @default false */
|
||||
skipped?: boolean
|
||||
}
|
||||
|
||||
type OnboardingResolver = (context: AudioInputPreflightContext) => OnboardingConfiguration | undefined | Promise<OnboardingConfiguration | undefined>
|
||||
|
||||
/** Configures onboarding with values selected by one case. */
|
||||
export function configureOnboarding(resolve: OnboardingResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureStorage(context.runtime, {
|
||||
'onboarding/completed': String(configuration.completed),
|
||||
'onboarding/skipped': String(configuration.skipped ?? false),
|
||||
})
|
||||
|
||||
if (configuration.completed || configuration.skipped) {
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 1_000))
|
||||
const onboardingPages = context.runtime.electronApp
|
||||
?.windows()
|
||||
.filter(page => new URL(page.url()).hash.startsWith('#/onboarding')) ?? []
|
||||
await Promise.all(onboardingPages.map(page => page.close().catch(() => undefined)))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
/** Provider values stored by one case preflight callback. */
|
||||
export interface ProviderConfiguration {
|
||||
config: Record<string, unknown>
|
||||
definitionId: string
|
||||
id: string
|
||||
model: string
|
||||
}
|
||||
|
||||
/** Adds one Provider without replacing Providers configured by earlier callbacks. */
|
||||
export async function configureProvider(runtime: AudioInputSession, provider: ProviderConfiguration): Promise<void> {
|
||||
await runtime.page.evaluate(({ configuredProvider }) => {
|
||||
const credentials = JSON.parse(localStorage.getItem('settings/credentials/providers') ?? '{}') as Record<string, unknown>
|
||||
const configured = JSON.parse(localStorage.getItem('settings/providers/configured') ?? '{}') as Record<string, unknown>
|
||||
const added = JSON.parse(localStorage.getItem('settings/providers/added') ?? '{}') as Record<string, boolean>
|
||||
|
||||
credentials[configuredProvider.id] = configuredProvider.config
|
||||
configured[configuredProvider.id] = {
|
||||
id: configuredProvider.id,
|
||||
definitionId: configuredProvider.definitionId,
|
||||
config: configuredProvider.config,
|
||||
status: 'configured',
|
||||
}
|
||||
added[configuredProvider.id] = true
|
||||
|
||||
localStorage.setItem('settings/credentials/providers', JSON.stringify(credentials))
|
||||
localStorage.setItem('settings/providers/configured', JSON.stringify(configured))
|
||||
localStorage.setItem('settings/providers/added', JSON.stringify(added))
|
||||
}, { configuredProvider: provider })
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
/** Writes AIRI settings through the storage owned by the current runtime page. */
|
||||
export async function configureStorage(
|
||||
runtime: AudioInputSession,
|
||||
settings: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await runtime.runtimePage.evaluate(({ entries }) => {
|
||||
for (const [key, value] of Object.entries(entries))
|
||||
localStorage.setItem(key, value)
|
||||
}, { entries: settings })
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { Locator } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
import { captureStreamingTranscription } from './streaming-transcription'
|
||||
|
||||
export interface EnableChatMicrophoneOptions {
|
||||
/** @default 'vad' */
|
||||
readiness?: 'streaming-transcription' | 'vad'
|
||||
}
|
||||
|
||||
/** Enables the chat microphone through the UI owned by the selected runtime. */
|
||||
export async function enableChatMicrophone(
|
||||
runtime: AudioInputSession,
|
||||
options: EnableChatMicrophoneOptions = {},
|
||||
): Promise<void> {
|
||||
if (runtime.target === 'electron') {
|
||||
const app = runtime.electronApp
|
||||
if (!app)
|
||||
throw new Error('The Electron audio session does not expose its application.')
|
||||
|
||||
const existingChatPage = app.windows().find(page => page.url().includes('index.html#/chat'))
|
||||
if (existingChatPage) {
|
||||
runtime.activatePage(existingChatPage)
|
||||
}
|
||||
else {
|
||||
const chatButton = runtime.runtimePage.locator('button').filter({
|
||||
has: runtime.runtimePage.locator('[i-solar\\:chat-line-line-duotone]'),
|
||||
}).first()
|
||||
await chatButton.waitFor({ state: 'visible', timeout: 30_000 })
|
||||
runtime.activatePage(await openElectronChat(app, chatButton))
|
||||
}
|
||||
}
|
||||
|
||||
const { page } = runtime
|
||||
await page.locator('textarea').first().waitFor({ state: 'visible', timeout: 60_000 })
|
||||
await captureStreamingTranscription(page, 'textarea')
|
||||
|
||||
if (runtime.target === 'electron') {
|
||||
await runtime.runtimePage.waitForFunction(async () => {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices()
|
||||
return devices.some(device => device.kind === 'audioinput' && device.label.includes('Fake'))
|
||||
})
|
||||
|
||||
const hearingTrigger = runtime.runtimePage.locator('div[aria-haspopup="dialog"] button').first()
|
||||
await hearingTrigger.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await hearingTrigger.hover()
|
||||
await hearingTrigger.click({ force: true })
|
||||
await runtime.runtimePage.waitForTimeout(500)
|
||||
|
||||
const enableButton = runtime.runtimePage.locator('button[aria-label="Enable microphone input"]')
|
||||
await enableButton.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
const inputReady = options.readiness === 'streaming-transcription'
|
||||
? runtime.waitForStreamingTranscriptionReady()
|
||||
: runtime.waitForVadReady()
|
||||
await enableButton.click({ force: true })
|
||||
const disableButton = runtime.runtimePage.locator('button[aria-label="Disable microphone input"]')
|
||||
await disableButton.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await inputReady
|
||||
return
|
||||
}
|
||||
|
||||
const microphoneTrigger = page.locator('button').filter({ has: page.locator('.i-ph\\:microphone-slash') }).first()
|
||||
await microphoneTrigger.click({ force: true })
|
||||
|
||||
const enableButton = page.locator('button[aria-label="Enable microphone input"]')
|
||||
await enableButton.waitFor({ state: 'visible' })
|
||||
const inputReady = options.readiness === 'streaming-transcription'
|
||||
? runtime.waitForStreamingTranscriptionReady()
|
||||
: runtime.waitForVadReady()
|
||||
await enableButton.click()
|
||||
await page.locator('button[aria-label="Disable microphone input"]').waitFor({ state: 'visible' })
|
||||
await inputReady
|
||||
}
|
||||
|
||||
/** Opens the chat page and waits for its input. */
|
||||
export async function openChat(runtime: AudioInputSession): Promise<void> {
|
||||
await runtime.page.evaluate(() => {
|
||||
window.location.hash = '/chat'
|
||||
})
|
||||
await runtime.page.locator('textarea').first().waitFor({ state: 'visible', timeout: 60_000 })
|
||||
}
|
||||
|
||||
/** Returns the assistant messages on the current chat page. */
|
||||
export function assistantMessages(runtime: AudioInputSession): Locator {
|
||||
return runtime.page.locator('[data-chat-message-role="assistant"] .markdown-content')
|
||||
}
|
||||
|
||||
async function openElectronChat(
|
||||
app: NonNullable<AudioInputSession['electronApp']>,
|
||||
chatButton: Locator,
|
||||
): Promise<AudioInputSession['page']> {
|
||||
let lastError: unknown
|
||||
for (let attempt = 0; attempt < 10; attempt++) {
|
||||
await chatButton.click({ force: true })
|
||||
try {
|
||||
return await waitForElectronPage(app, page => page.url().includes('index.html#/chat'), 3_000)
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError
|
||||
}
|
||||
|
||||
async function waitForElectronPage(
|
||||
app: NonNullable<AudioInputSession['electronApp']>,
|
||||
predicate: (page: AudioInputSession['page']) => boolean,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<AudioInputSession['page']> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const page = app.windows().find(predicate)
|
||||
if (page) {
|
||||
await page.waitForLoadState('domcontentloaded')
|
||||
await page.bringToFront()
|
||||
await page.waitForTimeout(750)
|
||||
return page
|
||||
}
|
||||
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 100))
|
||||
}
|
||||
|
||||
throw new Error(`Timed out while waiting for the Electron chat window. Open pages: ${app.windows().map(page => page.url()).join(', ')}`)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
import { captureStreamingTranscription } from './streaming-transcription'
|
||||
|
||||
/** Opens the Electron hearing playground for a case that selects this input UI. */
|
||||
export async function openHearingPlayground(runtime: AudioInputSession): Promise<Page> {
|
||||
const app = runtime.electronApp
|
||||
if (!app)
|
||||
throw new Error('The hearing playground interaction requires an Electron runtime')
|
||||
|
||||
const settingsButton = runtime.page.getByRole('button', { name: /Open settings|打开设置/ }).last()
|
||||
if (!await settingsButton.isVisible().catch(() => false)) {
|
||||
await runtime.page.getByRole('button', { name: /Expand|展开/ }).last().click({ force: true })
|
||||
await settingsButton.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
}
|
||||
|
||||
await settingsButton.click({ force: true })
|
||||
const settingsPage = await waitForElectronPage(app, page => page.url().includes('index.html#/settings'))
|
||||
await settingsPage.evaluate(() => {
|
||||
window.location.hash = '/settings/modules/hearing'
|
||||
})
|
||||
await settingsPage.waitForURL(/#\/settings\/modules\/hearing/)
|
||||
await settingsPage.getByTestId('hearing-playground-monitor-toggle').waitFor({ state: 'visible', timeout: 60_000 })
|
||||
return settingsPage
|
||||
}
|
||||
|
||||
/** Enables microphone monitoring on an open hearing playground. */
|
||||
export async function enableHearingPlaygroundMicrophone(page: Page): Promise<void> {
|
||||
await captureStreamingTranscription(page, '[data-testid="hearing-playground-current"] p')
|
||||
|
||||
const modelBasedToggle = page.getByRole('switch').last()
|
||||
if (await modelBasedToggle.isChecked())
|
||||
await modelBasedToggle.click()
|
||||
|
||||
const monitorToggle = page.getByTestId('hearing-playground-monitor-toggle')
|
||||
await monitorToggle.click()
|
||||
}
|
||||
|
||||
/** Reads the requested number of final hearing playground transcripts. */
|
||||
export async function readHearingPlaygroundTranscriptions(page: Page, count: number): Promise<string[]> {
|
||||
const transcriptions = page.getByTestId('hearing-playground-transcript')
|
||||
await transcriptions.nth(count - 1).waitFor({ state: 'visible', timeout: 60_000 })
|
||||
const results = (await transcriptions.allTextContents()).toReversed()
|
||||
await page.evaluate((transcriptionResults) => {
|
||||
if (window.__airiAudioInputE2E)
|
||||
window.__airiAudioInputE2E.transcriptionResults = transcriptionResults
|
||||
}, results)
|
||||
return results
|
||||
}
|
||||
|
||||
async function waitForElectronPage(
|
||||
app: ElectronApplication,
|
||||
predicate: (page: Page) => boolean,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<Page> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const page = app.windows().find(predicate)
|
||||
if (page) {
|
||||
await page.waitForLoadState('domcontentloaded')
|
||||
return page
|
||||
}
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 100))
|
||||
}
|
||||
throw new Error('Timed out while waiting for the Electron renderer')
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export { assistantMessages, enableChatMicrophone, openChat } from './chat'
|
||||
export type { EnableChatMicrophoneOptions } from './chat'
|
||||
export {
|
||||
enableHearingPlaygroundMicrophone,
|
||||
openHearingPlayground,
|
||||
readHearingPlaygroundTranscriptions,
|
||||
} from './hearing-playground'
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { Page } from 'playwright'
|
||||
|
||||
/** Records distinct visible transcription values until the page closes. */
|
||||
export async function captureStreamingTranscription(page: Page, selector: string): Promise<void> {
|
||||
await page.evaluate((captureSelector) => {
|
||||
let lastUpdate = ''
|
||||
setInterval(() => {
|
||||
const element = document.querySelector(captureSelector)
|
||||
const value = element instanceof HTMLTextAreaElement
|
||||
? element.value
|
||||
: element?.textContent
|
||||
const update = value?.trim() ?? ''
|
||||
const updates = window.__airiAudioInputE2E?.streamingTranscriptionUpdates
|
||||
if (update && update !== lastUpdate && updates) {
|
||||
lastUpdate = update
|
||||
updates.push(update)
|
||||
}
|
||||
}, 20)
|
||||
}, selector)
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { ProviderConfiguration } from '../configurations/provider'
|
||||
|
||||
/** Credentials and Provider selection for Aliyun NLS transcription. */
|
||||
export interface AliyunNlsAsrOptions {
|
||||
accessKeyId: string
|
||||
accessKeySecret: string
|
||||
appKey: string
|
||||
provider: string
|
||||
}
|
||||
|
||||
/** Creates an Aliyun NLS realtime ASR Provider configuration. */
|
||||
export function aliyunNlsAsr(options: AliyunNlsAsrOptions): ProviderConfiguration {
|
||||
if (options.provider !== 'aliyun-nls-transcription')
|
||||
throw new Error('The Aliyun NLS Provider must be "aliyun-nls-transcription".')
|
||||
|
||||
return {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: 'aliyun-nls-v1',
|
||||
config: {
|
||||
accessKeyId: options.accessKeyId,
|
||||
accessKeySecret: options.accessKeySecret,
|
||||
appKey: options.appKey,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export { aliyunNlsAsr } from './aliyun-nls'
|
||||
export type { AliyunNlsAsrOptions } from './aliyun-nls'
|
||||
export { openaiAsr, openaiLlm, openaiTts } from './openai'
|
||||
export type { OpenAIProviderOptions, OpenAISpeechProviderOptions } from './openai'
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { ProviderConfiguration } from '../configurations/provider'
|
||||
|
||||
/** Configuration for an OpenAI-compatible Provider selected by one case. */
|
||||
export interface OpenAIProviderOptions {
|
||||
apiKey: string
|
||||
baseUrl: string
|
||||
model: string
|
||||
provider: string
|
||||
}
|
||||
|
||||
/** Configuration for an OpenAI-compatible speech Provider selected by one case. */
|
||||
export interface OpenAISpeechProviderOptions extends OpenAIProviderOptions {
|
||||
voice: string
|
||||
}
|
||||
|
||||
/** Creates an OpenAI-compatible ASR Provider configuration. */
|
||||
export function openaiAsr(options: OpenAIProviderOptions): ProviderConfiguration {
|
||||
return {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: options.model,
|
||||
config: {
|
||||
apiKey: options.apiKey,
|
||||
baseUrl: options.baseUrl,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates an OpenAI-compatible LLM Provider configuration. */
|
||||
export function openaiLlm(options: OpenAIProviderOptions): ProviderConfiguration {
|
||||
return {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: options.model,
|
||||
config: {
|
||||
apiKey: options.apiKey,
|
||||
baseUrl: options.baseUrl,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates an OpenAI-compatible TTS Provider configuration. */
|
||||
export function openaiTts(options: OpenAISpeechProviderOptions): { provider: ProviderConfiguration, voice: string } {
|
||||
return {
|
||||
voice: options.voice,
|
||||
provider: {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: options.model,
|
||||
config: {
|
||||
apiKey: options.apiKey,
|
||||
baseUrl: options.baseUrl,
|
||||
model: options.model,
|
||||
voice: options.voice,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { assistantMessages, enableChatMicrophone, openChat } from '../shared/interactions'
|
||||
import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
// An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format.
|
||||
// The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence.
|
||||
// Its warm-up phrase gives the VAD time to start. Only "Please say hello." is required in the transcript.
|
||||
it('runs an OpenAI-compatible request through the complete pipeline', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
// This case keeps AIRI's default VAD and selects every remote Provider explicitly.
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
configureModuleConsciousness(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_LLM_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_LLM_API_KEY to run this LLM case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiLlm({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_LLM_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_LLM_MODEL ?? 'gpt-4o-mini',
|
||||
provider: environment.TESTING_AUDIO_LLM_PROVIDER ?? 'openai-compatible',
|
||||
}),
|
||||
}
|
||||
}),
|
||||
configureModuleSpeech(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_TTS_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_TTS_API_KEY to run this TTS case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return openaiTts({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_TTS_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_TTS_MODEL ?? 'tts-1',
|
||||
provider: environment.TESTING_AUDIO_TTS_PROVIDER ?? 'openai-compatible-audio-speech',
|
||||
voice: environment.TESTING_AUDIO_TTS_VOICE ?? 'alloy',
|
||||
})
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
|
||||
if (audio.transcriptionCaptureFormat) {
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
}
|
||||
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
], { match: 'contains' })
|
||||
|
||||
await expect.poll(async () => (await audio.completedSpans('LLM inference')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
await expect.poll(async () => (await audio.completedSpans('TTS synthesis')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
await expect.poll(async () => (await audio.completedSpans('Audio playback')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
|
||||
await openChat(audio)
|
||||
const messages = await assistantMessages(audio).allTextContents()
|
||||
expect(messages.at(-1)).toMatch(/.+/s)
|
||||
})
|
||||
|
||||
it('transcribes the greeting with Aliyun NLS', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
|
||||
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
|
||||
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
|
||||
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
|
||||
context.skip(
|
||||
!provider || !accessKeyId || !accessKeySecret || !appKey,
|
||||
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
|
||||
)
|
||||
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
|
||||
captureFormat: 'pcm',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
|
||||
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
], { match: 'contains' })
|
||||
})
|
||||
})
|
||||
Binary file not shown.
@@ -0,0 +1,105 @@
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { enableChatMicrophone } from '../shared/interactions'
|
||||
import { aliyunNlsAsr, openaiAsr } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
// The fixture has 12 seconds of leading silence so that the browser VAD can load.
|
||||
// It repeats the greeting after a 2-second pause to catch regressions that drop the second utterance.
|
||||
it('keeps two utterances in streaming transcription', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
// This regression isolates AIRI's default VAD and one explicit ASR Provider.
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
|
||||
if (audio.transcriptionCaptureFormat) {
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
}
|
||||
|
||||
const expectedTranscriptions = [
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
]
|
||||
await expect(audio).toHaveTranscriptions(expectedTranscriptions)
|
||||
|
||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps two Aliyun NLS utterances in streaming transcription', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
|
||||
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
|
||||
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
|
||||
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
|
||||
context.skip(
|
||||
!provider || !accessKeyId || !accessKeySecret || !appKey,
|
||||
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
|
||||
)
|
||||
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
|
||||
captureFormat: 'pcm',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
|
||||
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
const expectedTranscriptions = [
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
]
|
||||
await expect(audio).toHaveTranscriptions(expectedTranscriptions)
|
||||
|
||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||
})
|
||||
})
|
||||
Binary file not shown.
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"name": "@proj-airi/testing-audio",
|
||||
"type": "module",
|
||||
"version": "0.11.3",
|
||||
"private": true,
|
||||
"description": "AIRI Web and Electron audio pipeline tests",
|
||||
"author": {
|
||||
"name": "Moeru AI Project AIRI Team",
|
||||
"email": "airi@moeru.ai",
|
||||
"url": "https://github.com/moeru-ai"
|
||||
},
|
||||
"license": "MIT",
|
||||
"exports": "./src/index.ts",
|
||||
"scripts": {
|
||||
"build:targets": "pnpm -F @proj-airi/stage-web build && pnpm -F @proj-airi/stage-tamagotchi build",
|
||||
"test": "vitest",
|
||||
"test:run": "pnpm run build:targets && vitest run",
|
||||
"test:existing-builds": "vitest run",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@moeru/std": "catalog:",
|
||||
"@pnpm/find-workspace-dir": "catalog:",
|
||||
"@proj-airi/stage-shared": "workspace:^",
|
||||
"@proj-airi/stage-ui": "workspace:^",
|
||||
"@proj-airi/vitest-plugin-fakemic": "workspace:^",
|
||||
"playwright": "catalog:",
|
||||
"vite": "catalog:",
|
||||
"vitest": "catalog:vitest"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "catalog:"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { AudioTestTask } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputPreflightContext, AudioInputSession, AudioInputTestCase } from './types'
|
||||
|
||||
import { env } from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { createAudioTestAPI, createAudioTestTask, runAudioTestSession, startFakemicRuntime } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import { inject } from 'vitest'
|
||||
|
||||
import { expect, installAudioInputMatchers } from './expect-extend'
|
||||
|
||||
type RunnableAudioInputTest = AudioTestTask
|
||||
|
||||
installAudioInputMatchers()
|
||||
|
||||
const audioTestAPI = createAudioTestAPI<
|
||||
AudioInputTestCase,
|
||||
RunnableAudioInputTest,
|
||||
{ audio: AudioInputSession },
|
||||
AudioInputPreflightContext
|
||||
>({
|
||||
preflight: definition => definition.preflight,
|
||||
createPlans(name, testCase) {
|
||||
const task = createAudioTestTask(name, testCase)
|
||||
return [{
|
||||
name: task.name,
|
||||
definition: task,
|
||||
metadata: {
|
||||
input: fileURLToPath(task.input),
|
||||
runtime: inject('fakemicRuntime').name,
|
||||
},
|
||||
}]
|
||||
},
|
||||
async execute({ plan, task, invokeHandler, runPreflight }) {
|
||||
await runAudioTestSession({
|
||||
start() {
|
||||
const microphoneInput = fileURLToPath(plan.definition.input)
|
||||
return startFakemicRuntime<AudioInputSession>(microphoneInput)
|
||||
},
|
||||
async execute(session) {
|
||||
await runPreflight({
|
||||
env,
|
||||
runtime: session,
|
||||
skip: (condition, note) => task.context.skip(Boolean(condition), note),
|
||||
})
|
||||
await session.runtimePage.reload({ waitUntil: 'domcontentloaded' })
|
||||
await session.runtimePage.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await session.runtimePage.bringToFront()
|
||||
await session.runtimePage.waitForTimeout(750)
|
||||
Object.assign(task.context, { audio: session })
|
||||
await invokeHandler()
|
||||
},
|
||||
async recordArtifacts(session) {
|
||||
const snapshot = await session.snapshot().catch(error => ({
|
||||
snapshotError: errorMessageFrom(error) ?? 'Unknown snapshot error',
|
||||
}))
|
||||
await task.context.annotate('pipeline.json', {
|
||||
body: `${JSON.stringify(snapshot, null, 2)}\n`,
|
||||
bodyEncoding: 'utf-8',
|
||||
contentType: 'application/json',
|
||||
})
|
||||
},
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
/** Groups AIRI audio-input tests in the Vitest task tree. */
|
||||
export const describe = audioTestAPI.describe
|
||||
|
||||
/** Defines an AIRI audio-input test for each selected target. */
|
||||
export const it = audioTestAPI.it
|
||||
|
||||
/** Vitest expect with AIRI audio-input matchers installed. */
|
||||
export { expect }
|
||||
@@ -0,0 +1,28 @@
|
||||
import type { AudioInputObservations } from './types'
|
||||
|
||||
import { describe, it } from 'vitest'
|
||||
|
||||
import { expect, installAudioInputMatchers } from './expect-extend'
|
||||
|
||||
installAudioInputMatchers()
|
||||
|
||||
describe('audio input matchers', () => {
|
||||
it('normalizes transcription case, width, punctuation, and whitespace', async () => {
|
||||
const session = createAudioInputSession(['Ｐlease, SAY hello!'])
|
||||
|
||||
await expect(session).toHaveTranscriptions([
|
||||
['please say hello'],
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
function createAudioInputSession(transcriptions: string[]): AudioInputObservations {
|
||||
return {
|
||||
capturedTranscriptionAudio: async () => [],
|
||||
streamingTranscriptionUpdates: async () => [],
|
||||
transcriptionResults: async () => transcriptions,
|
||||
completedSpans: async () => [],
|
||||
waitForStreamingTranscriptionReady: async () => {},
|
||||
waitForVadReady: async () => {},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { AudioInputObservations } from './types'
|
||||
|
||||
import { expect as vitestExpect } from 'vitest'
|
||||
|
||||
export interface CapturedTranscriptionAudioExpectation {
|
||||
count: number
|
||||
minimumBytes: number
|
||||
}
|
||||
|
||||
export interface TranscriptionExpectationOptions {
|
||||
/** @default 'exact' */
|
||||
match?: 'exact' | 'contains'
|
||||
}
|
||||
|
||||
declare module 'vitest' {
|
||||
interface Assertion<T> {
|
||||
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
|
||||
? (expected: CapturedTranscriptionAudioExpectation) => Promise<void>
|
||||
: never
|
||||
toHaveTranscriptions: T extends AudioInputObservations
|
||||
? (
|
||||
expected: ReadonlyArray<ReadonlyArray<string>>,
|
||||
options?: TranscriptionExpectationOptions,
|
||||
) => Promise<void>
|
||||
: never
|
||||
}
|
||||
}
|
||||
|
||||
/** Vitest expect with AIRI audio-input matcher types. */
|
||||
export const expect = vitestExpect
|
||||
|
||||
/**
|
||||
* Normalizes a transcript for speech-recognition comparison.
|
||||
*
|
||||
* @example
|
||||
* normalizeTranscript(' Hello, AIRI! ')
|
||||
* // => 'helloairi'
|
||||
*/
|
||||
function normalizeTranscript(value: string): string {
|
||||
return value
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, '')
|
||||
}
|
||||
|
||||
/** Installs asynchronous matchers for AIRI audio-input observations. */
|
||||
export function installAudioInputMatchers(): void {
|
||||
vitestExpect.extend({
|
||||
async toHaveCapturedTranscriptionAudio(
|
||||
session: AudioInputObservations,
|
||||
expected: CapturedTranscriptionAudioExpectation,
|
||||
) {
|
||||
const format = session.transcriptionCaptureFormat
|
||||
if (!format) {
|
||||
return {
|
||||
pass: false,
|
||||
message: () => 'The active transcription Provider does not expose uploaded audio.',
|
||||
}
|
||||
}
|
||||
|
||||
const captures = await session.capturedTranscriptionAudio(expected.count)
|
||||
const invalidCapture = captures.find((capture) => {
|
||||
if (capture.format !== format || capture.data.byteLength < expected.minimumBytes)
|
||||
return true
|
||||
return format === 'wav' && new TextDecoder().decode(capture.data.subarray(0, 4)) !== 'RIFF'
|
||||
})
|
||||
const pass = captures.length === expected.count && !invalidCapture
|
||||
|
||||
return {
|
||||
pass,
|
||||
message: () => pass
|
||||
? 'Expected the session not to contain valid transcription audio.'
|
||||
: `Expected ${expected.count} ${format} capture(s) with at least ${expected.minimumBytes} bytes.`,
|
||||
}
|
||||
},
|
||||
async toHaveTranscriptions(
|
||||
session: AudioInputObservations,
|
||||
expected: ReadonlyArray<ReadonlyArray<string>>,
|
||||
options: TranscriptionExpectationOptions = {},
|
||||
) {
|
||||
const actual = await session.transcriptionResults(expected.length)
|
||||
const normalizedActual = actual.map(normalizeTranscript)
|
||||
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript))
|
||||
const match = options.match ?? 'exact'
|
||||
const pass = normalizedActual.length === normalizedExpected.length
|
||||
&& normalizedActual.every((transcript, index) => (
|
||||
match === 'contains'
|
||||
? normalizedExpected[index].some(candidate => transcript.includes(candidate))
|
||||
: normalizedExpected[index].includes(transcript)
|
||||
))
|
||||
|
||||
return {
|
||||
pass,
|
||||
message: () => pass
|
||||
? 'Expected the session not to contain the specified transcriptions.'
|
||||
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export { describe, expect, it } from './describe'
|
||||
|
||||
export type {
|
||||
AudioInputObservations,
|
||||
AudioInputPreflightCallback,
|
||||
AudioInputPreflightContext,
|
||||
AudioInputSession,
|
||||
AudioInputTarget,
|
||||
AudioInputTestCase,
|
||||
} from './types'
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { FakemicElectronPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
|
||||
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
|
||||
await context.app.context().addInitScript(stubForBrowser)
|
||||
const page = await waitForPage(context, (page) => {
|
||||
const url = new URL(page.url())
|
||||
return url.pathname.endsWith('/index.html') && url.hash === '#/'
|
||||
})
|
||||
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await page.evaluate(stubForBrowser)
|
||||
|
||||
return createSession({
|
||||
electronApp: context.app,
|
||||
page,
|
||||
target: 'electron',
|
||||
close: context.close,
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForPage(
|
||||
context: FakemicElectronPrepareContext,
|
||||
predicate: (page: Page) => boolean,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<Page> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const page = context.app.windows().find(predicate)
|
||||
if (page) {
|
||||
await page.waitForLoadState('domcontentloaded')
|
||||
return page
|
||||
}
|
||||
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 100))
|
||||
}
|
||||
|
||||
throw new Error('Timed out while waiting for the Electron renderer')
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
|
||||
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
|
||||
await context.context.addInitScript(stubForBrowser)
|
||||
|
||||
const page = await context.context.newPage()
|
||||
await page.goto(context.runtime.url)
|
||||
|
||||
return createSession({
|
||||
page,
|
||||
target: 'web',
|
||||
close: context.close,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
|
||||
/**
|
||||
* Installs passive browser probes before the application starts.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `BrowserContext.addInitScript`
|
||||
* -> {@link stubForBrowser}
|
||||
* -> `window.fetch`
|
||||
* -> `BroadcastChannel('io-tracer-channel')`
|
||||
*
|
||||
* Upstream:
|
||||
* - `BrowserContext.addInitScript`
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E`
|
||||
*/
|
||||
export function stubForBrowser() {
|
||||
const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
|
||||
window.__airiAudioInputE2E = state
|
||||
|
||||
const originalConsoleInfo = console.info.bind(console)
|
||||
console.info = (...values: unknown[]) => {
|
||||
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
|
||||
state.vadReady = true
|
||||
originalConsoleInfo(...values)
|
||||
}
|
||||
|
||||
const originalFetch = window.fetch.bind(window)
|
||||
|
||||
/**
|
||||
* Copies each ASR upload and response while it forwards the request.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `window.fetch`
|
||||
* -> `POST /audio/transcriptions`
|
||||
* -> captureFetch
|
||||
*
|
||||
* Upstream:
|
||||
* - The OpenAI-compatible transcription Provider.
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E`
|
||||
* - The original `window.fetch` function.
|
||||
*/
|
||||
const captureFetch: typeof window.fetch = async (input, init) => {
|
||||
const requestUrl = typeof input === 'string'
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.href
|
||||
: input.url
|
||||
|
||||
const capturesTranscription = new URL(requestUrl, window.location.href).pathname.endsWith('/audio/transcriptions')
|
||||
if (capturesTranscription && init?.body instanceof FormData) {
|
||||
const file = init.body.get('file')
|
||||
if (file instanceof Blob) {
|
||||
state.transcriptionAudio.push({
|
||||
base64: new Uint8Array(await file.arrayBuffer()).toBase64(),
|
||||
format: 'wav',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const response = await originalFetch(input, init)
|
||||
if (capturesTranscription && response.ok) {
|
||||
const payload = await response.clone().json() as { text?: unknown }
|
||||
if (typeof payload.text === 'string')
|
||||
state.transcriptionResults.push(payload.text)
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
window.fetch = captureFetch
|
||||
|
||||
const OriginalWebSocket = window.WebSocket
|
||||
const audioChunksBySocket = new WeakMap<WebSocket, Uint8Array[]>()
|
||||
const capturesAliyunNlsBySocket = new WeakMap<WebSocket, boolean>()
|
||||
const capturedSockets = new WeakSet<WebSocket>()
|
||||
|
||||
function capturePcmAudio(socket: WebSocket) {
|
||||
const audioChunks = audioChunksBySocket.get(socket)
|
||||
if (!audioChunks || capturedSockets.has(socket) || !audioChunks.length)
|
||||
return
|
||||
|
||||
const byteLength = audioChunks.reduce((total, chunk) => total + chunk.byteLength, 0)
|
||||
if (byteLength < 8192)
|
||||
return
|
||||
|
||||
capturedSockets.add(socket)
|
||||
const audio = new Uint8Array(byteLength)
|
||||
let offset = 0
|
||||
for (const chunk of audioChunks) {
|
||||
audio.set(chunk, offset)
|
||||
offset += chunk.byteLength
|
||||
}
|
||||
state.transcriptionAudio.push({ base64: audio.toBase64(), format: 'pcm' })
|
||||
}
|
||||
|
||||
class CaptureAliyunNlsWebSocket extends OriginalWebSocket {
|
||||
constructor(url: string | URL, protocols?: string | string[]) {
|
||||
super(url, protocols)
|
||||
const target = new URL(url.toString(), window.location.href)
|
||||
const capturesAliyunNls = target.hostname.startsWith('nls-gateway-') && target.hostname.endsWith('.aliyuncs.com')
|
||||
capturesAliyunNlsBySocket.set(this, capturesAliyunNls)
|
||||
|
||||
if (capturesAliyunNls) {
|
||||
audioChunksBySocket.set(this, [])
|
||||
this.addEventListener('open', () => {
|
||||
state.streamingTranscriptionReady = true
|
||||
})
|
||||
this.addEventListener('message', (event) => {
|
||||
if (typeof event.data !== 'string')
|
||||
return
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(event.data) as { header?: { name?: string }, payload?: { result?: unknown } }
|
||||
if (payload.header?.name === 'SentenceEnd' && typeof payload.payload?.result === 'string')
|
||||
state.transcriptionResults.push(payload.payload.result)
|
||||
}
|
||||
catch {
|
||||
// NLS can send non-transcription frames. The Provider handles those frames.
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
|
||||
const audioChunks = audioChunksBySocket.get(this)
|
||||
if (capturesAliyunNlsBySocket.get(this) && audioChunks && typeof data !== 'string') {
|
||||
if (data instanceof Blob) {
|
||||
void data.arrayBuffer().then((buffer) => {
|
||||
audioChunks.push(new Uint8Array(buffer))
|
||||
})
|
||||
}
|
||||
else if (ArrayBuffer.isView(data)) {
|
||||
audioChunks.push(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
|
||||
}
|
||||
else {
|
||||
audioChunks.push(new Uint8Array(data))
|
||||
}
|
||||
capturePcmAudio(this)
|
||||
}
|
||||
|
||||
let outboundData: string | Blob | BufferSource
|
||||
if (typeof data === 'string' || data instanceof Blob || data instanceof ArrayBuffer) {
|
||||
outboundData = data
|
||||
}
|
||||
else if (ArrayBuffer.isView(data)) {
|
||||
// WebSocket does not accept views backed by SharedArrayBuffer. The copy uses a regular ArrayBuffer.
|
||||
outboundData = new Uint8Array(data.buffer, data.byteOffset, data.byteLength).slice()
|
||||
}
|
||||
else {
|
||||
// ArrayBufferLike also includes SharedArrayBuffer. The copy keeps the original bytes in a supported buffer.
|
||||
outboundData = new Uint8Array(data).slice()
|
||||
}
|
||||
|
||||
super.send(outboundData)
|
||||
}
|
||||
|
||||
close(code?: number, reason?: string): void {
|
||||
const audioChunks = audioChunksBySocket.get(this)
|
||||
if (capturesAliyunNlsBySocket.get(this) && audioChunks && !capturedSockets.has(this) && audioChunks.length) {
|
||||
capturePcmAudio(this)
|
||||
}
|
||||
|
||||
super.close(code, reason)
|
||||
}
|
||||
}
|
||||
|
||||
window.WebSocket = CaptureAliyunNlsWebSocket
|
||||
|
||||
const channel = new BroadcastChannel('io-tracer-channel')
|
||||
|
||||
/**
|
||||
* Stores completed I/O spans for the case artifact.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `BroadcastChannel('io-tracer-channel')`
|
||||
* -> `message`
|
||||
* -> captureSpan
|
||||
*
|
||||
* Upstream:
|
||||
* - The AIRI I/O trace exporter.
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E.spans`
|
||||
*/
|
||||
const captureSpan = (event: MessageEvent) => {
|
||||
if (event.data?.type === 'span' && event.data.span?.ended)
|
||||
state.spans.push(event.data.span as SerializedIOSpan)
|
||||
}
|
||||
|
||||
channel.addEventListener('message', captureSpan)
|
||||
}
|
||||
|
||||
/** Returns the completed browser spans that match the optional span name. */
|
||||
export function readCompletedSpans(name?: string) {
|
||||
const spans = window.__airiAudioInputE2E?.spans ?? []
|
||||
return name ? spans.filter(span => span.name === name) : spans
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession, AudioInputTarget } from '../types'
|
||||
|
||||
import { Buffer } from 'node:buffer'
|
||||
|
||||
import { readCompletedSpans } from './browser-probe'
|
||||
|
||||
/** Creates the runtime session and records its page diagnostics. */
|
||||
export function createSession(options: {
|
||||
electronApp?: ElectronApplication
|
||||
page: Page
|
||||
target: AudioInputTarget
|
||||
close: () => Promise<void>
|
||||
transcriptionCaptureFormat?: AudioCapture['format']
|
||||
}): AudioInputSession {
|
||||
const diagnostics: string[] = []
|
||||
const observedPages = new WeakSet<Page>()
|
||||
|
||||
function observePage(page: Page) {
|
||||
if (observedPages.has(page))
|
||||
return
|
||||
|
||||
observedPages.add(page)
|
||||
page.on('console', (message) => {
|
||||
const text = message.text()
|
||||
const isAudioPipelineInfo = message.type() === 'info'
|
||||
&& (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription'))
|
||||
if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo)
|
||||
diagnostics.push(`[console:${message.type()}] ${text}`)
|
||||
})
|
||||
page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`))
|
||||
}
|
||||
|
||||
observePage(options.page)
|
||||
|
||||
const session: AudioInputSession = {
|
||||
electronApp: options.electronApp,
|
||||
page: options.page,
|
||||
runtimePage: options.page,
|
||||
target: options.target,
|
||||
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
|
||||
activatePage(page) {
|
||||
observePage(page)
|
||||
session.page = page
|
||||
},
|
||||
async capturedTranscriptionAudio(count) {
|
||||
try {
|
||||
await options.page.waitForFunction(expectedCount => (
|
||||
(window.__airiAudioInputE2E?.transcriptionAudio.length ?? 0) >= expectedCount
|
||||
), count, { timeout: 60_000 })
|
||||
}
|
||||
catch (error) {
|
||||
const runtimeState = await options.page.evaluate(async () => ({
|
||||
activeModel: localStorage.getItem('settings/hearing/active-model'),
|
||||
activeProvider: localStorage.getItem('settings/hearing/active-provider'),
|
||||
devices: (await navigator.mediaDevices.enumerateDevices()).map(device => ({
|
||||
deviceId: device.deviceId,
|
||||
kind: device.kind,
|
||||
label: device.label,
|
||||
})),
|
||||
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
|
||||
microphoneInput: localStorage.getItem('settings/audio/input'),
|
||||
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
|
||||
probeInstalled: Boolean(window.__airiAudioInputE2E),
|
||||
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
|
||||
url: window.location.href,
|
||||
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
|
||||
}))
|
||||
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error })
|
||||
}
|
||||
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
|
||||
return capturedAudio.map(audio => ({
|
||||
format: audio.format,
|
||||
data: Buffer.from(audio.base64, 'base64'),
|
||||
}))
|
||||
},
|
||||
streamingTranscriptionUpdates: () => session.page.evaluate(() => window.__airiAudioInputE2E?.streamingTranscriptionUpdates ?? []),
|
||||
async transcriptionResults(count) {
|
||||
await options.page.waitForFunction(expectedCount => (
|
||||
(window.__airiAudioInputE2E?.transcriptionResults.length ?? 0) >= expectedCount
|
||||
), count, { timeout: 60_000 })
|
||||
return options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionResults ?? [])
|
||||
},
|
||||
async completedSpans(name) {
|
||||
const runtimeSpans = await options.page.evaluate(readCompletedSpans, name)
|
||||
if (session.page === options.page)
|
||||
return runtimeSpans
|
||||
|
||||
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
|
||||
return [...runtimeSpans, ...interactionSpans]
|
||||
},
|
||||
async waitForVadReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async waitForStreamingTranscriptionReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async snapshot() {
|
||||
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
const interactionState = session.page === options.page
|
||||
? runtimeState
|
||||
: await session.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
return {
|
||||
spans: runtimeState?.spans ?? [],
|
||||
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
|
||||
transcriptionResults: runtimeState?.transcriptionResults ?? [],
|
||||
diagnostics: [...diagnostics],
|
||||
}
|
||||
},
|
||||
close: options.close,
|
||||
}
|
||||
|
||||
return session
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
export const audioInputTargets = ['web', 'electron'] as const
|
||||
|
||||
export type AudioInputTarget = (typeof audioInputTargets)[number]
|
||||
|
||||
/** Values available when one case resolves its preflight callbacks. */
|
||||
export interface AudioInputPreflightContext {
|
||||
/** Environment variables loaded for this case. */
|
||||
env: Readonly<NodeJS.ProcessEnv>
|
||||
/** Clean runtime that the configuration callback can prepare. */
|
||||
runtime: AudioInputSession
|
||||
/** Skips this case when its environment does not satisfy a case constraint. */
|
||||
skip: (condition: unknown, note?: string) => void
|
||||
}
|
||||
|
||||
/** One optional case callback that resolves configuration from its environment. */
|
||||
export type AudioInputPreflightCallback = AudioTestPreflightCallback<AudioInputPreflightContext>
|
||||
|
||||
/** One AIRI audio-input test definition. */
|
||||
export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
|
||||
|
||||
/** Snapshot of the observable AIRI audio pipeline state. */
|
||||
export interface AudioInputSnapshot {
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionUpdates: string[]
|
||||
transcriptionResults: string[]
|
||||
diagnostics: string[]
|
||||
}
|
||||
|
||||
/** Observable audio values used by AIRI matchers. */
|
||||
export interface AudioInputObservations {
|
||||
/** Capture format used by the active transcription Provider. */
|
||||
transcriptionCaptureFormat?: AudioCaptureFormat
|
||||
capturedTranscriptionAudio: (count: number) => Promise<AudioCapture[]>
|
||||
streamingTranscriptionUpdates: () => Promise<string[]>
|
||||
transcriptionResults: (count: number) => Promise<string[]>
|
||||
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
|
||||
/** Waits until the VAD audio graph is connected to the microphone stream. */
|
||||
waitForVadReady: () => Promise<void>
|
||||
/** Waits until a streaming transcription transport accepts microphone audio. */
|
||||
waitForStreamingTranscriptionReady: () => Promise<void>
|
||||
}
|
||||
|
||||
/** Runtime handle for one AIRI audio-input test. */
|
||||
export interface AudioInputSession extends AudioInputObservations, AudioTestSession {
|
||||
/** Electron application for Electron tasks. */
|
||||
electronApp?: ElectronApplication
|
||||
/** Page used by case interactions. */
|
||||
page: Page
|
||||
/** Page that owns the audio-input pipeline. */
|
||||
runtimePage: Page
|
||||
/** Runtime selected for this concrete task. */
|
||||
target: AudioInputTarget
|
||||
/** Selects the page used by subsequent case interactions. */
|
||||
activatePage: (page: Page) => void
|
||||
snapshot: () => Promise<AudioInputSnapshot>
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
|
||||
declare global {
|
||||
interface BrowserAudioInputState {
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionReady: boolean
|
||||
streamingTranscriptionUpdates: string[]
|
||||
transcriptionAudio: Array<{ base64: string, format: 'pcm' | 'wav' }>
|
||||
transcriptionResults: string[]
|
||||
vadReady: boolean
|
||||
}
|
||||
|
||||
interface Window {
|
||||
__airiAudioInputE2E?: BrowserAudioInputState
|
||||
}
|
||||
|
||||
// NOTICE:
|
||||
// TypeScript 5.9 does not declare the Baseline 2025 Uint8Array Base64 methods.
|
||||
// The test runs in current Playwright and Electron Chromium runtimes that implement this API.
|
||||
// Source: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array/toBase64
|
||||
// Remove this declaration when the TypeScript standard library includes the method.
|
||||
interface Uint8Array<TArrayBuffer extends ArrayBufferLike = ArrayBufferLike> {
|
||||
toBase64: (options?: {
|
||||
alphabet?: 'base64' | 'base64url'
|
||||
omitPadding?: boolean
|
||||
}) => string
|
||||
}
|
||||
}
|
||||
|
||||
export {}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"lib": [
|
||||
"ESNext",
|
||||
"DOM",
|
||||
"DOM.Iterable"
|
||||
],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"types": [
|
||||
"node"
|
||||
],
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"isolatedModules": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"cases/**/*.ts",
|
||||
"src/**/*.ts",
|
||||
"vitest.config.ts"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { join } from 'node:path'
|
||||
|
||||
import fakemic, { electron, web } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
projects: [
|
||||
fakemic({
|
||||
name: 'audio-web',
|
||||
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.web.test.ts'],
|
||||
runtime: web({
|
||||
name: 'web',
|
||||
prepare: new URL('./src/runtimes/prepare-web.ts', import.meta.url).href,
|
||||
url: 'http://127.0.0.1:4173/',
|
||||
context: { permissions: ['microphone'] },
|
||||
preview: {
|
||||
configFile: join(import.meta.dirname, '../../apps/stage-web/vite.config.ts'),
|
||||
root: join(import.meta.dirname, '../../apps/stage-web'),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
fakemic({
|
||||
name: 'audio-electron',
|
||||
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.electron.test.ts'],
|
||||
runtime: electron({
|
||||
name: 'electron',
|
||||
prepare: new URL('./src/runtimes/prepare-electron.ts', import.meta.url).href,
|
||||
entry: join(import.meta.dirname, '../../apps/stage-tamagotchi/out/main/index.js'),
|
||||
args: ['--no-sandbox'],
|
||||
cwd: join(import.meta.dirname, '../..'),
|
||||
temporaryUserData: {
|
||||
env: 'APP_USER_DATA_PATH',
|
||||
prefix: 'airi-testing-audio-',
|
||||
},
|
||||
}),
|
||||
}),
|
||||
{
|
||||
extends: true,
|
||||
test: {
|
||||
name: 'unit',
|
||||
include: ['src/**/*.test.ts'],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
Reference in New Issue
Block a user