feat(testing-audio): now entire audio input/output pipeline can be tested
This commit is contained in:
Neko Ayaka
@@ -0,0 +1,76 @@
|
||||
import type { AudioTestTask } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputPreflightContext, AudioInputSession, AudioInputTestCase } from './types'
|
||||
|
||||
import { env } from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { createAudioTestAPI, createAudioTestTask, runAudioTestSession, startFakemicRuntime } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import { inject } from 'vitest'
|
||||
|
||||
import { expect, installAudioInputMatchers } from './expect-extend'
|
||||
|
||||
type RunnableAudioInputTest = AudioTestTask
|
||||
|
||||
installAudioInputMatchers()
|
||||
|
||||
const audioTestAPI = createAudioTestAPI<
|
||||
AudioInputTestCase,
|
||||
RunnableAudioInputTest,
|
||||
{ audio: AudioInputSession },
|
||||
AudioInputPreflightContext
|
||||
>({
|
||||
preflight: definition => definition.preflight,
|
||||
createPlans(name, testCase) {
|
||||
const task = createAudioTestTask(name, testCase)
|
||||
return [{
|
||||
name: task.name,
|
||||
definition: task,
|
||||
metadata: {
|
||||
input: fileURLToPath(task.input),
|
||||
runtime: inject('fakemicRuntime').name,
|
||||
},
|
||||
}]
|
||||
},
|
||||
async execute({ plan, task, invokeHandler, runPreflight }) {
|
||||
await runAudioTestSession({
|
||||
start() {
|
||||
const microphoneInput = fileURLToPath(plan.definition.input)
|
||||
return startFakemicRuntime<AudioInputSession>(microphoneInput)
|
||||
},
|
||||
async execute(session) {
|
||||
await runPreflight({
|
||||
env,
|
||||
runtime: session,
|
||||
skip: (condition, note) => task.context.skip(Boolean(condition), note),
|
||||
})
|
||||
await session.runtimePage.reload({ waitUntil: 'domcontentloaded' })
|
||||
await session.runtimePage.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await session.runtimePage.bringToFront()
|
||||
await session.runtimePage.waitForTimeout(750)
|
||||
Object.assign(task.context, { audio: session })
|
||||
await invokeHandler()
|
||||
},
|
||||
async recordArtifacts(session) {
|
||||
const snapshot = await session.snapshot().catch(error => ({
|
||||
snapshotError: errorMessageFrom(error) ?? 'Unknown snapshot error',
|
||||
}))
|
||||
await task.context.annotate('pipeline.json', {
|
||||
body: `${JSON.stringify(snapshot, null, 2)}\n`,
|
||||
bodyEncoding: 'utf-8',
|
||||
contentType: 'application/json',
|
||||
})
|
||||
},
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
/** Groups AIRI audio-input tests in the Vitest task tree. */
|
||||
export const describe = audioTestAPI.describe
|
||||
|
||||
/** Defines an AIRI audio-input test for each selected target. */
|
||||
export const it = audioTestAPI.it
|
||||
|
||||
/** Vitest expect with AIRI audio-input matchers installed. */
|
||||
export { expect }
|
||||
@@ -0,0 +1,28 @@
|
||||
import type { AudioInputObservations } from './types'
|
||||
|
||||
import { describe, it } from 'vitest'
|
||||
|
||||
import { expect, installAudioInputMatchers } from './expect-extend'
|
||||
|
||||
installAudioInputMatchers()
|
||||
|
||||
describe('audio input matchers', () => {
|
||||
it('normalizes transcription case, width, punctuation, and whitespace', async () => {
|
||||
const session = createAudioInputSession(['Ｐlease, SAY hello!'])
|
||||
|
||||
await expect(session).toHaveTranscriptions([
|
||||
['please say hello'],
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
function createAudioInputSession(transcriptions: string[]): AudioInputObservations {
|
||||
return {
|
||||
capturedTranscriptionAudio: async () => [],
|
||||
streamingTranscriptionUpdates: async () => [],
|
||||
transcriptionResults: async () => transcriptions,
|
||||
completedSpans: async () => [],
|
||||
waitForStreamingTranscriptionReady: async () => {},
|
||||
waitForVadReady: async () => {},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { AudioInputObservations } from './types'
|
||||
|
||||
import { expect as vitestExpect } from 'vitest'
|
||||
|
||||
export interface CapturedTranscriptionAudioExpectation {
|
||||
count: number
|
||||
minimumBytes: number
|
||||
}
|
||||
|
||||
export interface TranscriptionExpectationOptions {
|
||||
/** @default 'exact' */
|
||||
match?: 'exact' | 'contains'
|
||||
}
|
||||
|
||||
declare module 'vitest' {
|
||||
interface Assertion<T> {
|
||||
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
|
||||
? (expected: CapturedTranscriptionAudioExpectation) => Promise<void>
|
||||
: never
|
||||
toHaveTranscriptions: T extends AudioInputObservations
|
||||
? (
|
||||
expected: ReadonlyArray<ReadonlyArray<string>>,
|
||||
options?: TranscriptionExpectationOptions,
|
||||
) => Promise<void>
|
||||
: never
|
||||
}
|
||||
}
|
||||
|
||||
/** Vitest expect with AIRI audio-input matcher types. */
|
||||
export const expect = vitestExpect
|
||||
|
||||
/**
|
||||
* Normalizes a transcript for speech-recognition comparison.
|
||||
*
|
||||
* @example
|
||||
* normalizeTranscript(' Hello, AIRI! ')
|
||||
* // => 'helloairi'
|
||||
*/
|
||||
function normalizeTranscript(value: string): string {
|
||||
return value
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, '')
|
||||
}
|
||||
|
||||
/** Installs asynchronous matchers for AIRI audio-input observations. */
|
||||
export function installAudioInputMatchers(): void {
|
||||
vitestExpect.extend({
|
||||
async toHaveCapturedTranscriptionAudio(
|
||||
session: AudioInputObservations,
|
||||
expected: CapturedTranscriptionAudioExpectation,
|
||||
) {
|
||||
const format = session.transcriptionCaptureFormat
|
||||
if (!format) {
|
||||
return {
|
||||
pass: false,
|
||||
message: () => 'The active transcription Provider does not expose uploaded audio.',
|
||||
}
|
||||
}
|
||||
|
||||
const captures = await session.capturedTranscriptionAudio(expected.count)
|
||||
const invalidCapture = captures.find((capture) => {
|
||||
if (capture.format !== format || capture.data.byteLength < expected.minimumBytes)
|
||||
return true
|
||||
return format === 'wav' && new TextDecoder().decode(capture.data.subarray(0, 4)) !== 'RIFF'
|
||||
})
|
||||
const pass = captures.length === expected.count && !invalidCapture
|
||||
|
||||
return {
|
||||
pass,
|
||||
message: () => pass
|
||||
? 'Expected the session not to contain valid transcription audio.'
|
||||
: `Expected ${expected.count} ${format} capture(s) with at least ${expected.minimumBytes} bytes.`,
|
||||
}
|
||||
},
|
||||
async toHaveTranscriptions(
|
||||
session: AudioInputObservations,
|
||||
expected: ReadonlyArray<ReadonlyArray<string>>,
|
||||
options: TranscriptionExpectationOptions = {},
|
||||
) {
|
||||
const actual = await session.transcriptionResults(expected.length)
|
||||
const normalizedActual = actual.map(normalizeTranscript)
|
||||
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript))
|
||||
const match = options.match ?? 'exact'
|
||||
const pass = normalizedActual.length === normalizedExpected.length
|
||||
&& normalizedActual.every((transcript, index) => (
|
||||
match === 'contains'
|
||||
? normalizedExpected[index].some(candidate => transcript.includes(candidate))
|
||||
: normalizedExpected[index].includes(transcript)
|
||||
))
|
||||
|
||||
return {
|
||||
pass,
|
||||
message: () => pass
|
||||
? 'Expected the session not to contain the specified transcriptions.'
|
||||
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export { describe, expect, it } from './describe'
|
||||
|
||||
export type {
|
||||
AudioInputObservations,
|
||||
AudioInputPreflightCallback,
|
||||
AudioInputPreflightContext,
|
||||
AudioInputSession,
|
||||
AudioInputTarget,
|
||||
AudioInputTestCase,
|
||||
} from './types'
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { FakemicElectronPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
|
||||
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
|
||||
await context.app.context().addInitScript(stubForBrowser)
|
||||
const page = await waitForPage(context, (page) => {
|
||||
const url = new URL(page.url())
|
||||
return url.pathname.endsWith('/index.html') && url.hash === '#/'
|
||||
})
|
||||
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await page.evaluate(stubForBrowser)
|
||||
|
||||
return createSession({
|
||||
electronApp: context.app,
|
||||
page,
|
||||
target: 'electron',
|
||||
close: context.close,
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForPage(
|
||||
context: FakemicElectronPrepareContext,
|
||||
predicate: (page: Page) => boolean,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<Page> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const page = context.app.windows().find(predicate)
|
||||
if (page) {
|
||||
await page.waitForLoadState('domcontentloaded')
|
||||
return page
|
||||
}
|
||||
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 100))
|
||||
}
|
||||
|
||||
throw new Error('Timed out while waiting for the Electron renderer')
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
|
||||
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
|
||||
await context.context.addInitScript(stubForBrowser)
|
||||
|
||||
const page = await context.context.newPage()
|
||||
await page.goto(context.runtime.url)
|
||||
|
||||
return createSession({
|
||||
page,
|
||||
target: 'web',
|
||||
close: context.close,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
|
||||
/**
|
||||
* Installs passive browser probes before the application starts.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `BrowserContext.addInitScript`
|
||||
* -> {@link stubForBrowser}
|
||||
* -> `window.fetch`
|
||||
* -> `BroadcastChannel('io-tracer-channel')`
|
||||
*
|
||||
* Upstream:
|
||||
* - `BrowserContext.addInitScript`
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E`
|
||||
*/
|
||||
export function stubForBrowser() {
|
||||
const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
|
||||
window.__airiAudioInputE2E = state
|
||||
|
||||
const originalConsoleInfo = console.info.bind(console)
|
||||
console.info = (...values: unknown[]) => {
|
||||
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
|
||||
state.vadReady = true
|
||||
originalConsoleInfo(...values)
|
||||
}
|
||||
|
||||
const originalFetch = window.fetch.bind(window)
|
||||
|
||||
/**
|
||||
* Copies each ASR upload and response while it forwards the request.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `window.fetch`
|
||||
* -> `POST /audio/transcriptions`
|
||||
* -> captureFetch
|
||||
*
|
||||
* Upstream:
|
||||
* - The OpenAI-compatible transcription Provider.
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E`
|
||||
* - The original `window.fetch` function.
|
||||
*/
|
||||
const captureFetch: typeof window.fetch = async (input, init) => {
|
||||
const requestUrl = typeof input === 'string'
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.href
|
||||
: input.url
|
||||
|
||||
const capturesTranscription = new URL(requestUrl, window.location.href).pathname.endsWith('/audio/transcriptions')
|
||||
if (capturesTranscription && init?.body instanceof FormData) {
|
||||
const file = init.body.get('file')
|
||||
if (file instanceof Blob) {
|
||||
state.transcriptionAudio.push({
|
||||
base64: new Uint8Array(await file.arrayBuffer()).toBase64(),
|
||||
format: 'wav',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const response = await originalFetch(input, init)
|
||||
if (capturesTranscription && response.ok) {
|
||||
const payload = await response.clone().json() as { text?: unknown }
|
||||
if (typeof payload.text === 'string')
|
||||
state.transcriptionResults.push(payload.text)
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
window.fetch = captureFetch
|
||||
|
||||
const OriginalWebSocket = window.WebSocket
|
||||
const audioChunksBySocket = new WeakMap<WebSocket, Uint8Array[]>()
|
||||
const capturesAliyunNlsBySocket = new WeakMap<WebSocket, boolean>()
|
||||
const capturedSockets = new WeakSet<WebSocket>()
|
||||
|
||||
function capturePcmAudio(socket: WebSocket) {
|
||||
const audioChunks = audioChunksBySocket.get(socket)
|
||||
if (!audioChunks || capturedSockets.has(socket) || !audioChunks.length)
|
||||
return
|
||||
|
||||
const byteLength = audioChunks.reduce((total, chunk) => total + chunk.byteLength, 0)
|
||||
if (byteLength < 8192)
|
||||
return
|
||||
|
||||
capturedSockets.add(socket)
|
||||
const audio = new Uint8Array(byteLength)
|
||||
let offset = 0
|
||||
for (const chunk of audioChunks) {
|
||||
audio.set(chunk, offset)
|
||||
offset += chunk.byteLength
|
||||
}
|
||||
state.transcriptionAudio.push({ base64: audio.toBase64(), format: 'pcm' })
|
||||
}
|
||||
|
||||
class CaptureAliyunNlsWebSocket extends OriginalWebSocket {
|
||||
constructor(url: string | URL, protocols?: string | string[]) {
|
||||
super(url, protocols)
|
||||
const target = new URL(url.toString(), window.location.href)
|
||||
const capturesAliyunNls = target.hostname.startsWith('nls-gateway-') && target.hostname.endsWith('.aliyuncs.com')
|
||||
capturesAliyunNlsBySocket.set(this, capturesAliyunNls)
|
||||
|
||||
if (capturesAliyunNls) {
|
||||
audioChunksBySocket.set(this, [])
|
||||
this.addEventListener('open', () => {
|
||||
state.streamingTranscriptionReady = true
|
||||
})
|
||||
this.addEventListener('message', (event) => {
|
||||
if (typeof event.data !== 'string')
|
||||
return
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(event.data) as { header?: { name?: string }, payload?: { result?: unknown } }
|
||||
if (payload.header?.name === 'SentenceEnd' && typeof payload.payload?.result === 'string')
|
||||
state.transcriptionResults.push(payload.payload.result)
|
||||
}
|
||||
catch {
|
||||
// NLS can send non-transcription frames. The Provider handles those frames.
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
|
||||
const audioChunks = audioChunksBySocket.get(this)
|
||||
if (capturesAliyunNlsBySocket.get(this) && audioChunks && typeof data !== 'string') {
|
||||
if (data instanceof Blob) {
|
||||
void data.arrayBuffer().then((buffer) => {
|
||||
audioChunks.push(new Uint8Array(buffer))
|
||||
})
|
||||
}
|
||||
else if (ArrayBuffer.isView(data)) {
|
||||
audioChunks.push(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
|
||||
}
|
||||
else {
|
||||
audioChunks.push(new Uint8Array(data))
|
||||
}
|
||||
capturePcmAudio(this)
|
||||
}
|
||||
|
||||
let outboundData: string | Blob | BufferSource
|
||||
if (typeof data === 'string' || data instanceof Blob || data instanceof ArrayBuffer) {
|
||||
outboundData = data
|
||||
}
|
||||
else if (ArrayBuffer.isView(data)) {
|
||||
// WebSocket does not accept views backed by SharedArrayBuffer. The copy uses a regular ArrayBuffer.
|
||||
outboundData = new Uint8Array(data.buffer, data.byteOffset, data.byteLength).slice()
|
||||
}
|
||||
else {
|
||||
// ArrayBufferLike also includes SharedArrayBuffer. The copy keeps the original bytes in a supported buffer.
|
||||
outboundData = new Uint8Array(data).slice()
|
||||
}
|
||||
|
||||
super.send(outboundData)
|
||||
}
|
||||
|
||||
close(code?: number, reason?: string): void {
|
||||
const audioChunks = audioChunksBySocket.get(this)
|
||||
if (capturesAliyunNlsBySocket.get(this) && audioChunks && !capturedSockets.has(this) && audioChunks.length) {
|
||||
capturePcmAudio(this)
|
||||
}
|
||||
|
||||
super.close(code, reason)
|
||||
}
|
||||
}
|
||||
|
||||
window.WebSocket = CaptureAliyunNlsWebSocket
|
||||
|
||||
const channel = new BroadcastChannel('io-tracer-channel')
|
||||
|
||||
/**
|
||||
* Stores completed I/O spans for the case artifact.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `BroadcastChannel('io-tracer-channel')`
|
||||
* -> `message`
|
||||
* -> captureSpan
|
||||
*
|
||||
* Upstream:
|
||||
* - The AIRI I/O trace exporter.
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E.spans`
|
||||
*/
|
||||
const captureSpan = (event: MessageEvent) => {
|
||||
if (event.data?.type === 'span' && event.data.span?.ended)
|
||||
state.spans.push(event.data.span as SerializedIOSpan)
|
||||
}
|
||||
|
||||
channel.addEventListener('message', captureSpan)
|
||||
}
|
||||
|
||||
/** Returns the completed browser spans that match the optional span name. */
|
||||
export function readCompletedSpans(name?: string) {
|
||||
const spans = window.__airiAudioInputE2E?.spans ?? []
|
||||
return name ? spans.filter(span => span.name === name) : spans
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession, AudioInputTarget } from '../types'
|
||||
|
||||
import { Buffer } from 'node:buffer'
|
||||
|
||||
import { readCompletedSpans } from './browser-probe'
|
||||
|
||||
/** Creates the runtime session and records its page diagnostics. */
|
||||
export function createSession(options: {
|
||||
electronApp?: ElectronApplication
|
||||
page: Page
|
||||
target: AudioInputTarget
|
||||
close: () => Promise<void>
|
||||
transcriptionCaptureFormat?: AudioCapture['format']
|
||||
}): AudioInputSession {
|
||||
const diagnostics: string[] = []
|
||||
const observedPages = new WeakSet<Page>()
|
||||
|
||||
function observePage(page: Page) {
|
||||
if (observedPages.has(page))
|
||||
return
|
||||
|
||||
observedPages.add(page)
|
||||
page.on('console', (message) => {
|
||||
const text = message.text()
|
||||
const isAudioPipelineInfo = message.type() === 'info'
|
||||
&& (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription'))
|
||||
if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo)
|
||||
diagnostics.push(`[console:${message.type()}] ${text}`)
|
||||
})
|
||||
page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`))
|
||||
}
|
||||
|
||||
observePage(options.page)
|
||||
|
||||
const session: AudioInputSession = {
|
||||
electronApp: options.electronApp,
|
||||
page: options.page,
|
||||
runtimePage: options.page,
|
||||
target: options.target,
|
||||
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
|
||||
activatePage(page) {
|
||||
observePage(page)
|
||||
session.page = page
|
||||
},
|
||||
async capturedTranscriptionAudio(count) {
|
||||
try {
|
||||
await options.page.waitForFunction(expectedCount => (
|
||||
(window.__airiAudioInputE2E?.transcriptionAudio.length ?? 0) >= expectedCount
|
||||
), count, { timeout: 60_000 })
|
||||
}
|
||||
catch (error) {
|
||||
const runtimeState = await options.page.evaluate(async () => ({
|
||||
activeModel: localStorage.getItem('settings/hearing/active-model'),
|
||||
activeProvider: localStorage.getItem('settings/hearing/active-provider'),
|
||||
devices: (await navigator.mediaDevices.enumerateDevices()).map(device => ({
|
||||
deviceId: device.deviceId,
|
||||
kind: device.kind,
|
||||
label: device.label,
|
||||
})),
|
||||
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
|
||||
microphoneInput: localStorage.getItem('settings/audio/input'),
|
||||
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
|
||||
probeInstalled: Boolean(window.__airiAudioInputE2E),
|
||||
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
|
||||
url: window.location.href,
|
||||
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
|
||||
}))
|
||||
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error })
|
||||
}
|
||||
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
|
||||
return capturedAudio.map(audio => ({
|
||||
format: audio.format,
|
||||
data: Buffer.from(audio.base64, 'base64'),
|
||||
}))
|
||||
},
|
||||
streamingTranscriptionUpdates: () => session.page.evaluate(() => window.__airiAudioInputE2E?.streamingTranscriptionUpdates ?? []),
|
||||
async transcriptionResults(count) {
|
||||
await options.page.waitForFunction(expectedCount => (
|
||||
(window.__airiAudioInputE2E?.transcriptionResults.length ?? 0) >= expectedCount
|
||||
), count, { timeout: 60_000 })
|
||||
return options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionResults ?? [])
|
||||
},
|
||||
async completedSpans(name) {
|
||||
const runtimeSpans = await options.page.evaluate(readCompletedSpans, name)
|
||||
if (session.page === options.page)
|
||||
return runtimeSpans
|
||||
|
||||
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
|
||||
return [...runtimeSpans, ...interactionSpans]
|
||||
},
|
||||
async waitForVadReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async waitForStreamingTranscriptionReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async snapshot() {
|
||||
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
const interactionState = session.page === options.page
|
||||
? runtimeState
|
||||
: await session.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
return {
|
||||
spans: runtimeState?.spans ?? [],
|
||||
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
|
||||
transcriptionResults: runtimeState?.transcriptionResults ?? [],
|
||||
diagnostics: [...diagnostics],
|
||||
}
|
||||
},
|
||||
close: options.close,
|
||||
}
|
||||
|
||||
return session
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
export const audioInputTargets = ['web', 'electron'] as const
|
||||
|
||||
export type AudioInputTarget = (typeof audioInputTargets)[number]
|
||||
|
||||
/** Values available when one case resolves its preflight callbacks. */
|
||||
export interface AudioInputPreflightContext {
|
||||
/** Environment variables loaded for this case. */
|
||||
env: Readonly<NodeJS.ProcessEnv>
|
||||
/** Clean runtime that the configuration callback can prepare. */
|
||||
runtime: AudioInputSession
|
||||
/** Skips this case when its environment does not satisfy a case constraint. */
|
||||
skip: (condition: unknown, note?: string) => void
|
||||
}
|
||||
|
||||
/** One optional case callback that resolves configuration from its environment. */
|
||||
export type AudioInputPreflightCallback = AudioTestPreflightCallback<AudioInputPreflightContext>
|
||||
|
||||
/** One AIRI audio-input test definition. */
|
||||
export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
|
||||
|
||||
/** Snapshot of the observable AIRI audio pipeline state. */
|
||||
export interface AudioInputSnapshot {
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionUpdates: string[]
|
||||
transcriptionResults: string[]
|
||||
diagnostics: string[]
|
||||
}
|
||||
|
||||
/** Observable audio values used by AIRI matchers. */
|
||||
export interface AudioInputObservations {
|
||||
/** Capture format used by the active transcription Provider. */
|
||||
transcriptionCaptureFormat?: AudioCaptureFormat
|
||||
capturedTranscriptionAudio: (count: number) => Promise<AudioCapture[]>
|
||||
streamingTranscriptionUpdates: () => Promise<string[]>
|
||||
transcriptionResults: (count: number) => Promise<string[]>
|
||||
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
|
||||
/** Waits until the VAD audio graph is connected to the microphone stream. */
|
||||
waitForVadReady: () => Promise<void>
|
||||
/** Waits until a streaming transcription transport accepts microphone audio. */
|
||||
waitForStreamingTranscriptionReady: () => Promise<void>
|
||||
}
|
||||
|
||||
/** Runtime handle for one AIRI audio-input test. */
|
||||
export interface AudioInputSession extends AudioInputObservations, AudioTestSession {
|
||||
/** Electron application for Electron tasks. */
|
||||
electronApp?: ElectronApplication
|
||||
/** Page used by case interactions. */
|
||||
page: Page
|
||||
/** Page that owns the audio-input pipeline. */
|
||||
runtimePage: Page
|
||||
/** Runtime selected for this concrete task. */
|
||||
target: AudioInputTarget
|
||||
/** Selects the page used by subsequent case interactions. */
|
||||
activatePage: (page: Page) => void
|
||||
snapshot: () => Promise<AudioInputSnapshot>
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
|
||||
declare global {
|
||||
interface BrowserAudioInputState {
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionReady: boolean
|
||||
streamingTranscriptionUpdates: string[]
|
||||
transcriptionAudio: Array<{ base64: string, format: 'pcm' | 'wav' }>
|
||||
transcriptionResults: string[]
|
||||
vadReady: boolean
|
||||
}
|
||||
|
||||
interface Window {
|
||||
__airiAudioInputE2E?: BrowserAudioInputState
|
||||
}
|
||||
|
||||
// NOTICE:
|
||||
// TypeScript 5.9 does not declare the Baseline 2025 Uint8Array Base64 methods.
|
||||
// The test runs in current Playwright and Electron Chromium runtimes that implement this API.
|
||||
// Source: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array/toBase64
|
||||
// Remove this declaration when the TypeScript standard library includes the method.
|
||||
interface Uint8Array<TArrayBuffer extends ArrayBufferLike = ArrayBufferLike> {
|
||||
toBase64: (options?: {
|
||||
alphabet?: 'base64' | 'base64url'
|
||||
omitPadding?: boolean
|
||||
}) => string
|
||||
}
|
||||
}
|
||||
|
||||
export {}
|
||||
Reference in New Issue
Block a user