feat(stage-tamagotchi): add Apple Speech transcription (#2364)
This commit is contained in:
Neko
@@ -1,6 +1,46 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { toWav, toWavFromPCM16 } from './wav'
|
||||
import { toFloat32FromPCM16, toPCM16FromFloat32, toWav, toWavFromPCM16 } from './wav'
|
||||
|
||||
describe('pcm sample encoding', () => {
|
||||
it('converts normalized Float32 samples to little-endian PCM16 bytes', () => {
|
||||
const samples = new Float32Array([-2, -1, -0.5, 0, 0.5, 1, 2])
|
||||
const pcmBytes = toPCM16FromFloat32(samples)
|
||||
const view = new DataView(pcmBytes.buffer, pcmBytes.byteOffset, pcmBytes.byteLength)
|
||||
|
||||
expect(view.getInt16(0, true)).toBe(-32768)
|
||||
expect(view.getInt16(2, true)).toBe(-32768)
|
||||
expect(view.getInt16(4, true)).toBe(-16384)
|
||||
expect(view.getInt16(6, true)).toBe(0)
|
||||
expect(view.getInt16(8, true)).toBe(16383)
|
||||
expect(view.getInt16(10, true)).toBe(32767)
|
||||
expect(view.getInt16(12, true)).toBe(32767)
|
||||
})
|
||||
|
||||
it('converts little-endian PCM16 bytes to normalized Float32 samples', () => {
|
||||
const pcmBytes = new Uint8Array(10)
|
||||
const view = new DataView(pcmBytes.buffer)
|
||||
view.setInt16(0, -32768, true)
|
||||
view.setInt16(2, -16384, true)
|
||||
view.setInt16(4, 0, true)
|
||||
view.setInt16(6, 16384, true)
|
||||
view.setInt16(8, 32767, true)
|
||||
|
||||
expect([...toFloat32FromPCM16(pcmBytes)]).toEqual([
|
||||
-1,
|
||||
-0.5,
|
||||
0,
|
||||
0.5,
|
||||
32767 / 32768,
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects incomplete PCM16 samples', () => {
|
||||
expect(() => toFloat32FromPCM16(new Uint8Array([0]))).toThrow(
|
||||
'PCM16 input must contain complete 16-bit samples.',
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('toWav', () => {
|
||||
it('converts Float32 samples to PCM16 bytes by default', () => {
|
||||
|
||||
@@ -32,6 +32,47 @@ function createWavBuffer(dataSize: number, sampleRate: number, channel: number):
|
||||
return arrayBuffer
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts normalized Float32 PCM samples to little-endian signed PCM16 bytes.
|
||||
* Values outside the normalized range are clamped.
|
||||
*
|
||||
* @example
|
||||
* toPCM16FromFloat32(new Float32Array([-1, 0, 1]))
|
||||
* // => Uint8Array([0, 128, 0, 0, 255, 127])
|
||||
*/
|
||||
export function toPCM16FromFloat32(samples: Float32Array): Uint8Array<ArrayBuffer> {
|
||||
const output = new Uint8Array(samples.length * Int16Array.BYTES_PER_ELEMENT)
|
||||
const dataView = new DataView(output.buffer)
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const sample = Math.max(-1, Math.min(1, samples[i]))
|
||||
const value = sample < 0 ? sample * 0x8000 : sample * 0x7FFF
|
||||
dataView.setInt16(i * Int16Array.BYTES_PER_ELEMENT, value, true)
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts little-endian signed PCM16 bytes to normalized Float32 PCM samples.
|
||||
*
|
||||
* @example
|
||||
* toFloat32FromPCM16(new Uint8Array([0, 128, 0, 0, 255, 127]))
|
||||
* // => Float32Array([-1, 0, 0.999969482421875])
|
||||
*/
|
||||
export function toFloat32FromPCM16(pcmBytes: Uint8Array): Float32Array<ArrayBuffer> {
|
||||
if (pcmBytes.byteLength % Int16Array.BYTES_PER_ELEMENT !== 0)
|
||||
throw new TypeError('PCM16 input must contain complete 16-bit samples.')
|
||||
|
||||
const dataView = new DataView(pcmBytes.buffer, pcmBytes.byteOffset, pcmBytes.byteLength)
|
||||
const output = new Float32Array(pcmBytes.byteLength / Int16Array.BYTES_PER_ELEMENT)
|
||||
|
||||
for (let i = 0; i < output.length; i++)
|
||||
output[i] = dataView.getInt16(i * Int16Array.BYTES_PER_ELEMENT, true) / 0x8000
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes Float32 samples as a WAV file.
|
||||
*
|
||||
@@ -41,16 +82,7 @@ function createWavBuffer(dataSize: number, sampleRate: number, channel: number):
|
||||
*/
|
||||
export function toWav(buffer: ArrayBufferLike, sampleRate: number, channel = 1): ArrayBuffer {
|
||||
const samples = new Float32Array(buffer)
|
||||
const arrayBuffer = createWavBuffer(samples.length * 2, sampleRate, channel)
|
||||
const dataView = new DataView(arrayBuffer)
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const sample = Math.max(-1, Math.min(1, samples[i]))
|
||||
const value = sample < 0 ? sample * 0x8000 : sample * 0x7FFF
|
||||
dataView.setInt16(44 + i * 2, value, true)
|
||||
}
|
||||
|
||||
return arrayBuffer
|
||||
return toWavFromPCM16(toPCM16FromFloat32(samples), sampleRate, channel)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
},
|
||||
"inlinedDependencies": {
|
||||
"@electron-toolkit/preload": "3.0.2",
|
||||
"@moeru/eventa": "1.0.0-beta.15",
|
||||
"@moeru/eventa": "1.0.0",
|
||||
"async-mutex": "0.5.0",
|
||||
"nanoid": [
|
||||
"5.1.11",
|
||||
|
||||
@@ -941,6 +941,7 @@ pages:
|
||||
config:
|
||||
loading: Loading provider settings...
|
||||
load-error: Failed to load provider settings.
|
||||
save-error: Failed to save provider settings.
|
||||
retry: Retry
|
||||
common:
|
||||
fields:
|
||||
@@ -1082,6 +1083,14 @@ pages:
|
||||
OpenAI, Azure Speech
|
||||
description: LLMs, speech providers, etc.
|
||||
provider:
|
||||
apple-speech-transcription:
|
||||
title: Apple Speech
|
||||
description: On-device speech recognition on macOS 26 or later. No API key is required.
|
||||
fields:
|
||||
locale:
|
||||
label: Locale
|
||||
description: Use an exact Apple Speech locale, such as en-US or zh-CN.
|
||||
placeholder: en-US
|
||||
app-local-audio-transcription:
|
||||
title: App (Local)
|
||||
description: https://github.com/moeru-ai/xsai-transformers
|
||||
|
||||
@@ -903,6 +903,7 @@ pages:
|
||||
config:
|
||||
loading: 正在加载服务来源设置……
|
||||
load-error: 无法加载服务来源设置。
|
||||
save-error: 无法保存服务来源设置。
|
||||
retry: 重试
|
||||
common:
|
||||
fields:
|
||||
@@ -1031,6 +1032,14 @@ pages:
|
||||
转录（语音转文本）模型服务来源，例如 Whisper.cpp, OpenAI, Azure Speech
|
||||
description: LLM，语音合成，语音识别服务来源等
|
||||
provider:
|
||||
apple-speech-transcription:
|
||||
title: Apple 语音识别
|
||||
description: 使用 macOS 26 或更高版本的设备端语音识别，无需 API 密钥。
|
||||
fields:
|
||||
locale:
|
||||
label: 语言区域
|
||||
description: 使用 Apple 语音识别支持的完整语言区域代码，例如 en-US 或 zh-CN。
|
||||
placeholder: zh-CN
|
||||
app-local-audio-transcription:
|
||||
title: 应用内（本地）
|
||||
description: https://github.com/moeru-ai/xsai-transformers
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"dependencies": {
|
||||
"@moeru/eventa": "catalog:",
|
||||
"@moeru/std": "catalog:",
|
||||
"@proj-airi/audio": "workspace:^",
|
||||
"@proj-airi/ccc": "workspace:*",
|
||||
"@proj-airi/i18n": "workspace:*",
|
||||
"@proj-airi/server-sdk": "workspace:*",
|
||||
|
||||
+2
-10
@@ -3,6 +3,7 @@ import type { ServerEvent, ServerEvents } from '@proj-airi/stage-ui/libs/provide
|
||||
|
||||
import vadWorkletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url'
|
||||
|
||||
import { toPCM16FromFloat32 } from '@proj-airi/audio/encoding'
|
||||
import { errorMessageFromValue } from '@proj-airi/stage-shared'
|
||||
import { createAliyunNLSProvider } from '@proj-airi/stage-ui/libs/providers/providers/aliyun-nls'
|
||||
import { streamTranscription } from '@proj-airi/stage-ui/libs/providers/stream-transcription'
|
||||
@@ -83,15 +84,6 @@ function appendLog(message: string, level: 'info' | 'error' = 'info') {
|
||||
})
|
||||
}
|
||||
|
||||
function float32ToInt16(buffer: Float32Array) {
|
||||
const output = new Int16Array(buffer.length)
|
||||
for (let i = 0; i < buffer.length; i++) {
|
||||
const value = Math.max(-1, Math.min(1, buffer[i]))
|
||||
output[i] = value < 0 ? value * 0x8000 : value * 0x7FFF
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
function resetRecordingCounters() {
|
||||
audioChunkCount = 0
|
||||
lastChunkLogAt = 0
|
||||
@@ -116,7 +108,7 @@ async function initializeAudioGraph(stream: MediaStream) {
|
||||
if (!buffer || !controller)
|
||||
return
|
||||
|
||||
const pcm16 = float32ToInt16(buffer)
|
||||
const pcm16 = toPCM16FromFloat32(buffer)
|
||||
controller.enqueue(pcm16.buffer.slice(0))
|
||||
|
||||
audioChunkCount += 1
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { Alert, ErrorContainer, LevelMeter, RadioCardManySelect, RadioCardSimple, TestDummyMarker, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui/components'
|
||||
import { useAnalytics, useAudioAnalyzer, useHearingPlaygroundSegments, useVoiceInputSession } from '@proj-airi/stage-ui/composables'
|
||||
import { hearingProviderViewContextKey } from '@proj-airi/stage-ui/libs'
|
||||
import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
|
||||
import { CONFIDENCE_THRESHOLD_DISABLED, useHearingSpeechInputPipeline, useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
import { useProviderConfigStore } from '@proj-airi/stage-ui/stores/providers/config'
|
||||
@@ -9,7 +10,7 @@ import { useProviderStore } from '@proj-airi/stage-ui/stores/providers/provider'
|
||||
import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
|
||||
import { Button, FieldCheckbox, FieldCombobox, FieldInput, FieldRange } from '@proj-airi/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { computed, onMounted, onUnmounted, shallowRef, watch } from 'vue'
|
||||
import { computed, defineAsyncComponent, onMounted, onUnmounted, provide, shallowRef, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
import HearingPlaygroundTranscripts from './components/hearing-playground-transcripts.vue'
|
||||
@@ -59,6 +60,15 @@ let volumeSpeechEndTimer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const error = shallowRef('')
|
||||
const isMonitoring = shallowRef(false)
|
||||
const activeProviderConfig = computed(() => {
|
||||
if (!activeTranscriptionProvider.value)
|
||||
return undefined
|
||||
return providerStore.providers[activeTranscriptionProvider.value]?.config
|
||||
})
|
||||
const activeProviderHearingView = computed(() => {
|
||||
const loadView = providersStore.findProviderDefinition(activeTranscriptionProvider.value)?.views?.hearing
|
||||
return loadView ? defineAsyncComponent(loadView) : undefined
|
||||
})
|
||||
|
||||
const {
|
||||
current: currentTranscription,
|
||||
@@ -251,6 +261,50 @@ function updateCustomModelName(value: string | undefined) {
|
||||
activeTranscriptionModel.value = modelValue
|
||||
}
|
||||
|
||||
async function updateActiveProviderConfig(patch: Record<string, unknown>) {
|
||||
const providerId = activeTranscriptionProvider.value
|
||||
if (!providerId)
|
||||
throw new Error('No transcription Provider is active.')
|
||||
|
||||
const shouldRestartMonitoring = isMonitoring.value
|
||||
|
||||
try {
|
||||
await providersStore.initializeProvider(providerId)
|
||||
const provider = providerStore.getProvider(providerId)
|
||||
if (!provider)
|
||||
throw new Error('The transcription Provider configuration is unavailable.')
|
||||
|
||||
const update = providerStore.updateProviderConfig(
|
||||
providerId,
|
||||
{ ...provider.config, ...patch },
|
||||
'configured',
|
||||
)
|
||||
|
||||
if (shouldRestartMonitoring) {
|
||||
isMonitoring.value = false
|
||||
await stopAudioMonitoring(providerId)
|
||||
}
|
||||
|
||||
await update
|
||||
await providersStore.disposeProviderInstance(providerId)
|
||||
clearPlaygroundSegments()
|
||||
|
||||
// The selected Provider can change while a remote configuration save is pending.
|
||||
// Only restart the monitoring session for the Provider that requested the save.
|
||||
if (shouldRestartMonitoring && activeTranscriptionProvider.value === providerId)
|
||||
isMonitoring.value = await setupAudioMonitoring()
|
||||
}
|
||||
catch (cause) {
|
||||
error.value = errorMessageFrom(cause) ?? t('settings.pages.providers.catalog.edit.config.save-error')
|
||||
throw cause
|
||||
}
|
||||
}
|
||||
|
||||
provide(hearingProviderViewContextKey, {
|
||||
providerConfig: activeProviderConfig,
|
||||
updateProviderConfig: updateActiveProviderConfig,
|
||||
})
|
||||
|
||||
// Sync OpenAI Compatible model from provider config
|
||||
function syncOpenAICompatibleSettings() {
|
||||
if (activeTranscriptionProvider.value !== 'openai-compatible-audio-transcription')
|
||||
@@ -417,6 +471,11 @@ onUnmounted(() => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<component
|
||||
:is="activeProviderHearingView"
|
||||
v-if="activeProviderHearingView"
|
||||
/>
|
||||
|
||||
<!-- Model selection section -->
|
||||
<div v-if="activeTranscriptionProvider">
|
||||
<div flex="~ col gap-4">
|
||||
|
||||
+2
-10
@@ -6,6 +6,7 @@ import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/providers/
|
||||
|
||||
import vadWorkletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url'
|
||||
|
||||
import { toPCM16FromFloat32 } from '@proj-airi/audio/encoding'
|
||||
import { errorMessageFromValue } from '@proj-airi/stage-shared'
|
||||
import {
|
||||
Alert,
|
||||
@@ -123,15 +124,6 @@ const {
|
||||
forceValid,
|
||||
} = useProviderValidation(providerId)
|
||||
|
||||
function float32ToInt16(buffer: Float32Array) {
|
||||
const output = new Int16Array(buffer.length)
|
||||
for (let i = 0; i < buffer.length; i++) {
|
||||
const value = Math.max(-1, Math.min(1, buffer[i]))
|
||||
output[i] = value < 0 ? value * 0x8000 : value * 0x7FFF
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
async function initializeAudioGraph(stream: MediaStream) {
|
||||
const context = new AudioContext({
|
||||
sampleRate: SAMPLE_RATE,
|
||||
@@ -146,7 +138,7 @@ async function initializeAudioGraph(stream: MediaStream) {
|
||||
if (!buffer || !controller)
|
||||
return
|
||||
|
||||
const pcm16 = float32ToInt16(buffer)
|
||||
const pcm16 = toPCM16FromFloat32(buffer)
|
||||
controller.enqueue(pcm16.buffer.slice(0))
|
||||
}
|
||||
|
||||
|
||||
@@ -526,6 +526,15 @@ function handleDeleteProvider() {
|
||||
:required="field.required"
|
||||
@update:model-value="setFieldValue(field.key, $event)"
|
||||
/>
|
||||
<FieldCombobox
|
||||
v-else-if="field.type === 'select'"
|
||||
:model-value="getStringField(field.key)"
|
||||
:label="field.label"
|
||||
:description="field.description"
|
||||
:placeholder="field.placeholder"
|
||||
:options="field.options"
|
||||
@update:model-value="setFieldValue(field.key, $event)"
|
||||
/>
|
||||
<FieldInput
|
||||
v-else
|
||||
v-model="providerConfigEdit.config[field.key]"
|
||||
|
||||
@@ -105,6 +105,8 @@
|
||||
"@vueuse/core": "catalog:",
|
||||
"@vueuse/motion": "catalog:",
|
||||
"@vueuse/shared": "catalog:",
|
||||
"@xsai-apple-speech/transcription": "catalog:",
|
||||
"@xsai-apple-speech/transcription-electron-plugin": "catalog:",
|
||||
"@xsai-ext/providers": "catalog:",
|
||||
"@xsai-transformers/embed": "catalog:",
|
||||
"@xsai-transformers/shared": "catalog:",
|
||||
|
||||
@@ -9,6 +9,7 @@ import type { VoiceKey, Voices } from '../../../workers/kokoro/types'
|
||||
import type { AllocationToken } from '../gpu-resource-coordinator'
|
||||
import type { ProgressPayload } from '../protocol'
|
||||
|
||||
import { toWav } from '@proj-airi/audio/encoding'
|
||||
import { defaultPerfTracer } from '@proj-airi/stage-shared'
|
||||
import { Mutex } from 'async-mutex'
|
||||
|
||||
@@ -74,57 +75,6 @@ export interface KokoroAdapter {
|
||||
const LOAD_MODEL_TIMEOUT = TIMEOUTS.KOKORO_LOAD
|
||||
const GENERATE_TIMEOUT = TIMEOUTS.KOKORO_GENERATE
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audio Encoding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Encode raw PCM Float32Array samples into a WAV ArrayBuffer.
|
||||
* This runs on the main thread — intentionally lightweight (just header + int16 conversion).
|
||||
*/
|
||||
function encodeWav(samples: Float32Array, sampleRate: number, numChannels = 1): ArrayBuffer {
|
||||
const bitsPerSample = 16
|
||||
const bytesPerSample = bitsPerSample / 8
|
||||
const dataLength = samples.length * bytesPerSample
|
||||
const headerLength = 44
|
||||
const buffer = new ArrayBuffer(headerLength + dataLength)
|
||||
const view = new DataView(buffer)
|
||||
|
||||
// RIFF header
|
||||
writeString(view, 0, 'RIFF')
|
||||
view.setUint32(4, 36 + dataLength, true)
|
||||
writeString(view, 8, 'WAVE')
|
||||
|
||||
// fmt chunk
|
||||
writeString(view, 12, 'fmt ')
|
||||
view.setUint32(16, 16, true) // chunk size
|
||||
view.setUint16(20, 1, true) // PCM format
|
||||
view.setUint16(22, numChannels, true)
|
||||
view.setUint32(24, sampleRate, true)
|
||||
view.setUint32(28, sampleRate * numChannels * bytesPerSample, true) // byte rate
|
||||
view.setUint16(32, numChannels * bytesPerSample, true) // block align
|
||||
view.setUint16(34, bitsPerSample, true)
|
||||
|
||||
// data chunk
|
||||
writeString(view, 36, 'data')
|
||||
view.setUint32(40, dataLength, true)
|
||||
|
||||
// Convert Float32 [-1, 1] to Int16
|
||||
const output = new Int16Array(buffer, headerLength)
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, samples[i]))
|
||||
output[i] = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||||
}
|
||||
|
||||
return buffer
|
||||
}
|
||||
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -454,7 +404,7 @@ export function createKokoroAdapter(): KokoroAdapter {
|
||||
if (output.action === 'generate') {
|
||||
state = 'ready'
|
||||
onSuccess()
|
||||
return encodeWav(output.samples as Float32Array, output.samplingRate as number)
|
||||
return toWav((output.samples as Float32Array).buffer, output.samplingRate as number)
|
||||
}
|
||||
|
||||
const errorCode = classifyError(new Error('Unexpected output action'))
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { ComputedRef, InjectionKey } from 'vue'
|
||||
|
||||
import { inject } from 'vue'
|
||||
|
||||
/** APIs that the Hearing module exposes to the active Provider view. */
|
||||
export interface HearingProviderViewContext {
|
||||
/** Configuration for the active transcription Provider. */
|
||||
providerConfig: ComputedRef<Readonly<Record<string, unknown>> | undefined>
|
||||
/** Saves a partial configuration as configured and refreshes the monitoring session. */
|
||||
updateProviderConfig: (patch: Record<string, unknown>) => Promise<void>
|
||||
}
|
||||
|
||||
export const hearingProviderViewContextKey: InjectionKey<HearingProviderViewContext>
|
||||
= Symbol('hearing-provider-view-context')
|
||||
|
||||
/** Returns the Hearing module APIs available to a registered Provider view. */
|
||||
export function useHearingProviderViewContext() {
|
||||
const context = inject(hearingProviderViewContextKey)
|
||||
if (!context)
|
||||
throw new Error('The Provider view must be rendered inside the Hearing module.')
|
||||
|
||||
return context
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
export * from './attributes'
|
||||
export * from './hearing-view'
|
||||
export * from './metadata'
|
||||
export * from './providers'
|
||||
export * from './types'
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
<script setup lang="ts">
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { FieldCombobox, GhostButton } from '@proj-airi/ui'
|
||||
import { computedAsync } from '@vueuse/core'
|
||||
import { computed, shallowRef } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
import { useHearingProviderViewContext } from '../../hearing-view'
|
||||
import { listAppleSpeechLocaleOptions } from './provider'
|
||||
|
||||
const { t } = useI18n()
|
||||
const { providerConfig, updateProviderConfig } = useHearingProviderViewContext()
|
||||
|
||||
const loadAttempt = shallowRef(0)
|
||||
const isLoading = shallowRef(false)
|
||||
const isSaving = shallowRef(false)
|
||||
const loadError = shallowRef<string>()
|
||||
const saveError = shallowRef<string>()
|
||||
|
||||
const locale = computed(() => {
|
||||
const value = providerConfig.value?.locale
|
||||
return typeof value === 'string' && value.trim() ? value : 'en-US'
|
||||
})
|
||||
const localeOptions = computedAsync(async (onCancel) => {
|
||||
void loadAttempt.value
|
||||
|
||||
const abortController = new AbortController()
|
||||
onCancel(() => abortController.abort())
|
||||
loadError.value = undefined
|
||||
|
||||
try {
|
||||
return await listAppleSpeechLocaleOptions({
|
||||
abortSignal: abortController.signal,
|
||||
config: { locale: locale.value },
|
||||
t,
|
||||
})
|
||||
}
|
||||
catch (cause) {
|
||||
if (!abortController.signal.aborted) {
|
||||
loadError.value = errorMessageFrom(cause)
|
||||
?? t('settings.pages.providers.catalog.edit.config.load-error')
|
||||
}
|
||||
return []
|
||||
}
|
||||
}, [], { evaluating: isLoading })
|
||||
|
||||
function retry() {
|
||||
loadAttempt.value++
|
||||
}
|
||||
|
||||
async function updateLocale(value: string | undefined) {
|
||||
if (!value || value === locale.value || isSaving.value)
|
||||
return
|
||||
|
||||
isSaving.value = true
|
||||
saveError.value = undefined
|
||||
try {
|
||||
await updateProviderConfig({ locale: value })
|
||||
}
|
||||
catch (cause) {
|
||||
saveError.value = errorMessageFrom(cause)
|
||||
?? t('settings.pages.providers.catalog.edit.config.save-error')
|
||||
}
|
||||
finally {
|
||||
isSaving.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div :class="['flex', 'flex-col', 'gap-2']">
|
||||
<FieldCombobox
|
||||
data-testid="apple-speech-locale"
|
||||
:model-value="locale"
|
||||
:label="t('settings.pages.providers.provider.apple-speech-transcription.fields.locale.label')"
|
||||
:description="t('settings.pages.providers.provider.apple-speech-transcription.fields.locale.description')"
|
||||
:placeholder="isLoading
|
||||
? t('settings.pages.providers.catalog.edit.config.loading')
|
||||
: t('settings.pages.providers.provider.apple-speech-transcription.fields.locale.placeholder')"
|
||||
:options="localeOptions"
|
||||
:disabled="isLoading || isSaving || !!loadError"
|
||||
layout="vertical"
|
||||
@update:model-value="updateLocale"
|
||||
>
|
||||
<template #label>
|
||||
<div :class="['flex', 'items-center', 'gap-2']">
|
||||
<span>{{ t('settings.pages.providers.provider.apple-speech-transcription.fields.locale.label') }}</span>
|
||||
<div v-if="isLoading || isSaving" :class="['i-svg-spinners:ring-resize', 'text-sm', 'text-neutral-400']" />
|
||||
</div>
|
||||
</template>
|
||||
</FieldCombobox>
|
||||
|
||||
<div
|
||||
v-if="loadError || saveError"
|
||||
role="alert"
|
||||
:class="[
|
||||
'flex', 'items-center', 'justify-between', 'gap-2',
|
||||
'text-xs', 'text-red-600', 'dark:text-red-400',
|
||||
]"
|
||||
>
|
||||
<span>{{ loadError || saveError }}</span>
|
||||
<GhostButton
|
||||
v-if="loadError"
|
||||
size="sm"
|
||||
:label="t('settings.pages.providers.catalog.edit.config.retry')"
|
||||
@click="retry"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -0,0 +1,150 @@
|
||||
import type { AppleSpeechSessionOperations, StartStreamTranscriptionOptions, TranscriptionResult } from '@xsai-apple-speech/transcription'
|
||||
import type { ZodObject } from 'zod'
|
||||
|
||||
import { createStreamTranscriptionResult } from '@xsai-apple-speech/transcription'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { executeAppleSpeechStream, providerAppleSpeechTranscription } from '.'
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
dispose: vi.fn(),
|
||||
getLocales: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@moeru/eventa/adapters/electron/renderer', () => ({
|
||||
createContext: () => ({ context: {}, dispose: mocks.dispose }),
|
||||
}))
|
||||
|
||||
vi.mock('@proj-airi/stage-shared', () => ({
|
||||
isElectronWindow: () => true,
|
||||
isStageTamagotchi: () => true,
|
||||
}))
|
||||
|
||||
vi.mock('@xsai-apple-speech/transcription-electron-plugin', () => ({
|
||||
createAppleSpeechProvider: () => ({
|
||||
getLocales: mocks.getLocales,
|
||||
}),
|
||||
}))
|
||||
|
||||
describe('apple speech transcription provider', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('loads labeled locale options with automatic transcriber selection', async () => {
|
||||
vi.stubGlobal('window', {
|
||||
electron: { ipcRenderer: {} },
|
||||
platform: 'darwin',
|
||||
})
|
||||
mocks.getLocales.mockResolvedValue([
|
||||
{ installed: false, locale: 'en-US' },
|
||||
{ installed: true, locale: 'zh-CN' },
|
||||
])
|
||||
|
||||
const schema = await providerAppleSpeechTranscription.createProviderConfig({
|
||||
t: input => input,
|
||||
config: { locale: 'zh-CN' },
|
||||
})
|
||||
const localeMeta = (schema as ZodObject).shape.locale.meta()
|
||||
|
||||
expect(mocks.getLocales).toHaveBeenCalledWith({ transcriber: 'automatic' })
|
||||
expect(localeMeta?.type).toBe('select')
|
||||
expect(localeMeta?.options).toEqual([
|
||||
expect.objectContaining({ value: 'zh-CN' }),
|
||||
expect.objectContaining({ value: 'en-US' }),
|
||||
])
|
||||
expect(localeMeta?.options).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ label: expect.stringContaining('zh-CN') }),
|
||||
expect.objectContaining({ label: expect.stringContaining('en-US') }),
|
||||
]))
|
||||
expect(mocks.dispose).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('converts PCM16 input and emits AIRI transcript snapshots', async () => {
|
||||
const writtenSamples: Float32Array[] = []
|
||||
const finalResult: TranscriptionResult = {
|
||||
locale: 'en-US',
|
||||
results: [{
|
||||
range: {
|
||||
durationMilliseconds: 400,
|
||||
isFinal: true,
|
||||
startMilliseconds: 100,
|
||||
},
|
||||
text: 'Hello, AIRI.',
|
||||
}],
|
||||
text: 'Hello, AIRI.',
|
||||
}
|
||||
|
||||
const result = executeAppleSpeechStream({
|
||||
baseURL: new URL('apple-speech://transcription'),
|
||||
fetch: globalThis.fetch,
|
||||
inputAudioStream: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Int16Array([-32768, -16384, 0, 16384, 32767]).buffer)
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
inputSampleRate: 16000,
|
||||
model: 'apple-speech',
|
||||
startStream: (streamOptions: StartStreamTranscriptionOptions) => createStreamTranscriptionResult({
|
||||
...streamOptions,
|
||||
locale: 'en-US',
|
||||
async start(request): Promise<AppleSpeechSessionOperations> {
|
||||
return {
|
||||
async dispose() {},
|
||||
async finish() {
|
||||
return finalResult
|
||||
},
|
||||
async write(samples) {
|
||||
writtenSamples.push(samples.slice())
|
||||
await request.onPartial({
|
||||
locale: 'en-US',
|
||||
range: {
|
||||
durationMilliseconds: 300,
|
||||
isFinal: false,
|
||||
startMilliseconds: 100,
|
||||
},
|
||||
text: 'Hello, Ari',
|
||||
type: 'transcript.text.partial',
|
||||
})
|
||||
},
|
||||
}
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
await expect(result.text).resolves.toBe('Hello, AIRI.')
|
||||
expect(writtenSamples).toHaveLength(1)
|
||||
expect(Array.from(writtenSamples[0] ?? [])).toEqual([
|
||||
-1,
|
||||
-0.5,
|
||||
0,
|
||||
0.5,
|
||||
32767 / 32768,
|
||||
])
|
||||
|
||||
const events = []
|
||||
for await (const event of result.fullStream)
|
||||
events.push(event)
|
||||
|
||||
expect(events).toEqual([
|
||||
{
|
||||
durationMilliseconds: 300,
|
||||
isFinal: false,
|
||||
locale: 'en-US',
|
||||
startMilliseconds: 100,
|
||||
text: 'Hello, Ari',
|
||||
type: 'transcript.text.snapshot',
|
||||
},
|
||||
{
|
||||
durationMilliseconds: 400,
|
||||
isFinal: true,
|
||||
locale: 'en-US',
|
||||
startMilliseconds: 100,
|
||||
text: 'Hello, AIRI.',
|
||||
type: 'transcript.text.snapshot',
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,248 @@
|
||||
import type {
|
||||
AppleSpeechTranscription,
|
||||
TranscriptionEvent,
|
||||
TranscriptionRange,
|
||||
} from '@xsai-apple-speech/transcription'
|
||||
import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/providers/utils'
|
||||
|
||||
import type { AIRIStreamTranscriptionResult, StreamTranscriptionOptions } from '../../stream-transcription'
|
||||
import type { ProviderConfigContext } from '../../types'
|
||||
import type { AppleSpeechConfig } from './provider'
|
||||
|
||||
import { createContext } from '@moeru/eventa/adapters/electron/renderer'
|
||||
import { toFloat32FromPCM16 } from '@proj-airi/audio/encoding'
|
||||
import { isElectronWindow, isStageTamagotchi } from '@proj-airi/stage-shared'
|
||||
import { streamTranscription as streamAppleSpeechTranscription } from '@xsai-apple-speech/transcription'
|
||||
import { createAppleSpeechProvider as createElectronAppleSpeechProvider } from '@xsai-apple-speech/transcription-electron-plugin'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
import { appleSpeechConfigSchema, listAppleSpeechLocaleOptions } from './provider'
|
||||
|
||||
export type { AppleSpeechConfig } from './provider'
|
||||
export { listAppleSpeechLocaleOptions } from './provider'
|
||||
|
||||
export const APPLE_SPEECH_TRANSCRIPTION_PROVIDER_ID = 'apple-speech-transcription'
|
||||
|
||||
/** Request options applied by AIRI before Apple Speech creates a batch or live session. */
|
||||
export interface AppleSpeechProviderOptions {
|
||||
/** Cancels native preparation and active transcription work. */
|
||||
abortSignal?: AbortSignal
|
||||
/** PCM input rate in hertz. @default 16000 */
|
||||
inputSampleRate?: number
|
||||
/** Exact Apple Speech locale for this request. The Provider configuration is the default. */
|
||||
locale?: string
|
||||
}
|
||||
|
||||
type AIRIAppleSpeechProvider = TranscriptionProviderWithExtraOptions<'apple-speech', AppleSpeechProviderOptions> & {
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
type AppleSpeechStreamOptions = StreamTranscriptionOptions & AppleSpeechTranscription & {
|
||||
inputSampleRate?: number
|
||||
}
|
||||
|
||||
async function createAppleSpeechConfigSchema(context: ProviderConfigContext<AppleSpeechConfig>) {
|
||||
const { t } = context
|
||||
const localeOptions = await listAppleSpeechLocaleOptions(context)
|
||||
return appleSpeechConfigSchema.extend({
|
||||
locale: appleSpeechConfigSchema.shape.locale.meta({
|
||||
type: 'select',
|
||||
labelLocalized: t('settings.pages.providers.provider.apple-speech-transcription.fields.locale.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.provider.apple-speech-transcription.fields.locale.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.provider.apple-speech-transcription.fields.locale.placeholder'),
|
||||
options: localeOptions,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
function createRendererAppleSpeechProvider(config: AppleSpeechConfig): AIRIAppleSpeechProvider {
|
||||
if (typeof window === 'undefined' || !isElectronWindow(window))
|
||||
throw new Error('Apple Speech transcription requires the Electron desktop app.')
|
||||
|
||||
const eventa = createContext(window.electron.ipcRenderer)
|
||||
const provider = createElectronAppleSpeechProvider({ context: eventa.context })
|
||||
const configuredLocale = config.locale?.trim() || 'en-US'
|
||||
|
||||
return {
|
||||
transcription(_model, requestOptions = {}) {
|
||||
const locale = requestOptions.locale?.trim() || configuredLocale
|
||||
return {
|
||||
...provider.transcription({ locale, transcriber: 'automatic' }),
|
||||
...requestOptions,
|
||||
inputSampleRate: requestOptions.inputSampleRate ?? 16000,
|
||||
}
|
||||
},
|
||||
dispose() {
|
||||
eventa.dispose()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function isAppleSpeechAvailable() {
|
||||
if (!isStageTamagotchi() || typeof window === 'undefined' || !isElectronWindow(window) || window.platform !== 'darwin')
|
||||
return false
|
||||
|
||||
const { context, dispose } = createContext(window.electron.ipcRenderer)
|
||||
|
||||
try {
|
||||
const provider = createElectronAppleSpeechProvider({ context })
|
||||
const availability = await provider.isAvailable()
|
||||
return availability.available
|
||||
}
|
||||
catch {
|
||||
return false
|
||||
}
|
||||
finally {
|
||||
dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function isAppleSpeechStreamRequest(
|
||||
options: StreamTranscriptionOptions,
|
||||
): options is AppleSpeechStreamOptions {
|
||||
return options.baseURL instanceof URL
|
||||
&& typeof options.fetch === 'function'
|
||||
&& 'model' in options
|
||||
&& typeof options.model === 'string'
|
||||
&& 'startStream' in options
|
||||
&& typeof options.startStream === 'function'
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes one PCM chunk to a byte view without copying its sample data.
|
||||
*
|
||||
* @example
|
||||
* audioChunkBytes(new Int16Array([0, 1]))
|
||||
* // => Uint8Array(4)
|
||||
*/
|
||||
function audioChunkBytes(chunk: ArrayBuffer | ArrayBufferView) {
|
||||
return ArrayBuffer.isView(chunk)
|
||||
? new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength)
|
||||
: new Uint8Array(chunk)
|
||||
}
|
||||
|
||||
function combinedRange(event: TranscriptionEvent): TranscriptionRange {
|
||||
if (event.type === 'transcript.text.partial')
|
||||
return event.range
|
||||
|
||||
const ranges = event.results?.map(result => result.range) ?? []
|
||||
if (ranges.length === 0)
|
||||
return { durationMilliseconds: 0, isFinal: true, startMilliseconds: 0 }
|
||||
|
||||
const startMilliseconds = Math.min(...ranges.map(range => range.startMilliseconds))
|
||||
const endMilliseconds = Math.max(...ranges.map(range => range.startMilliseconds + range.durationMilliseconds))
|
||||
return {
|
||||
durationMilliseconds: endMilliseconds - startMilliseconds,
|
||||
isFinal: true,
|
||||
startMilliseconds,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes one Apple replacement event to the snapshot contract used by Hearing.
|
||||
*
|
||||
* @example
|
||||
* appleEventSnapshot({ type: 'transcript.text.partial', text: 'Hello', locale: 'en-US', range })
|
||||
* // => { type: 'transcript.text.snapshot', text: 'Hello', isFinal: false, ... }
|
||||
*/
|
||||
function appleEventSnapshot(event: TranscriptionEvent) {
|
||||
const range = combinedRange(event)
|
||||
return {
|
||||
durationMilliseconds: range.durationMilliseconds,
|
||||
isFinal: event.type === 'transcript.text.done',
|
||||
locale: event.locale,
|
||||
startMilliseconds: range.startMilliseconds,
|
||||
text: event.text,
|
||||
type: 'transcript.text.snapshot' as const,
|
||||
}
|
||||
}
|
||||
|
||||
async function pumpPcm16Input(
|
||||
input: NonNullable<StreamTranscriptionOptions['inputAudioStream']>,
|
||||
writer: WritableStreamDefaultWriter<Float32Array>,
|
||||
) {
|
||||
const reader = input.getReader()
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done)
|
||||
break
|
||||
|
||||
await writer.write(toFloat32FromPCM16(audioChunkBytes(value)))
|
||||
}
|
||||
await writer.close()
|
||||
}
|
||||
catch (error) {
|
||||
await writer.abort(error).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
finally {
|
||||
reader.releaseLock()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapts AIRI's mono PCM16 VAD stream to Apple Speech live transcription.
|
||||
*
|
||||
* The Provider boundary converts each audio chunk and maps Apple replacement
|
||||
* events to AIRI transcript snapshots.
|
||||
*/
|
||||
export function executeAppleSpeechStream(options: AppleSpeechStreamOptions): AIRIStreamTranscriptionResult
|
||||
export function executeAppleSpeechStream(options: StreamTranscriptionOptions): AIRIStreamTranscriptionResult
|
||||
export function executeAppleSpeechStream(options: StreamTranscriptionOptions): AIRIStreamTranscriptionResult {
|
||||
if (!options.inputAudioStream)
|
||||
throw new TypeError('Apple Speech live transcription requires an audio stream.')
|
||||
if (!isAppleSpeechStreamRequest(options))
|
||||
throw new TypeError('Apple Speech live transcription requires a native stream request.')
|
||||
|
||||
const inputSampleRate = options.inputSampleRate ?? 16000
|
||||
const live = streamAppleSpeechTranscription({
|
||||
...options,
|
||||
inputSampleRate,
|
||||
})
|
||||
const inputPump = pumpPcm16Input(options.inputAudioStream, live.input.getWriter())
|
||||
void inputPump.catch(() => {})
|
||||
|
||||
return {
|
||||
fullStream: live.fullStream.pipeThrough(new TransformStream({
|
||||
transform(event, controller) {
|
||||
controller.enqueue(appleEventSnapshot(event))
|
||||
},
|
||||
})),
|
||||
text: Promise.all([live.text, inputPump]).then(([text]) => text),
|
||||
textStream: live.partialStream,
|
||||
}
|
||||
}
|
||||
|
||||
export const providerAppleSpeechTranscription = defineProvider<AppleSpeechConfig>({
|
||||
id: APPLE_SPEECH_TRANSCRIPTION_PROVIDER_ID,
|
||||
name: 'Apple Speech',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.apple-speech-transcription.title'),
|
||||
description: 'On-device speech recognition on macOS 26 or later. No API key is required.',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.apple-speech-transcription.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'],
|
||||
requiresCredentials: false,
|
||||
isAvailableBy: isAppleSpeechAvailable,
|
||||
views: {
|
||||
hearing: () => import('./hearing-settings.vue'),
|
||||
},
|
||||
capabilities: {
|
||||
transcription: {
|
||||
protocol: 'native',
|
||||
generateOutput: true,
|
||||
streamOutput: true,
|
||||
streamInput: true,
|
||||
},
|
||||
},
|
||||
createProviderConfig: createAppleSpeechConfigSchema,
|
||||
createProvider: createRendererAppleSpeechProvider,
|
||||
validationRequiredWhen: () => false,
|
||||
extraMethods: {
|
||||
listModels: async () => [{
|
||||
id: 'apple-speech',
|
||||
name: 'Apple Speech',
|
||||
provider: APPLE_SPEECH_TRANSCRIPTION_PROVIDER_ID,
|
||||
description: 'On-device Apple Speech transcription',
|
||||
}],
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { AppleSpeechLocale } from '@xsai-apple-speech/transcription'
|
||||
|
||||
import type { ProviderConfigContext } from '../../types'
|
||||
|
||||
import { createContext } from '@moeru/eventa/adapters/electron/renderer'
|
||||
import { isElectronWindow } from '@proj-airi/stage-shared'
|
||||
import { createAppleSpeechProvider } from '@xsai-apple-speech/transcription-electron-plugin'
|
||||
import { z } from 'zod'
|
||||
|
||||
export const appleSpeechConfigSchema = z.object({
|
||||
locale: z.string().trim().min(1).default('en-US'),
|
||||
})
|
||||
|
||||
/** Serializable configuration for the Apple Speech Provider. */
|
||||
export type AppleSpeechConfig = z.input<typeof appleSpeechConfigSchema>
|
||||
|
||||
function localeLabel(locale: string) {
|
||||
try {
|
||||
const displayName = new Intl.DisplayNames([locale], { type: 'language' }).of(locale)
|
||||
if (displayName && displayName !== locale)
|
||||
return `${displayName} (${locale})`
|
||||
}
|
||||
catch {
|
||||
// Apple owns this locale inventory. Keep its canonical identifier visible
|
||||
// if the current JavaScript runtime cannot format a newer language tag.
|
||||
}
|
||||
return locale
|
||||
}
|
||||
|
||||
function sortLocales(locales: AppleSpeechLocale[]) {
|
||||
return [...locales].sort((left, right) => {
|
||||
if (left.installed !== right.installed)
|
||||
return left.installed ? -1 : 1
|
||||
return left.locale.localeCompare(right.locale)
|
||||
})
|
||||
}
|
||||
|
||||
/** Lists labeled native locales available through Apple's automatic transcriber selection. */
|
||||
export async function listAppleSpeechLocaleOptions(context: ProviderConfigContext<AppleSpeechConfig>) {
|
||||
if (context.config === undefined)
|
||||
return []
|
||||
if (typeof window === 'undefined' || !isElectronWindow(window) || window.platform !== 'darwin')
|
||||
return []
|
||||
|
||||
context.abortSignal?.throwIfAborted()
|
||||
const eventa = createContext(window.electron.ipcRenderer)
|
||||
try {
|
||||
const provider = createAppleSpeechProvider({ context: eventa.context })
|
||||
const locales = await provider.getLocales({ transcriber: 'automatic' })
|
||||
context.abortSignal?.throwIfAborted()
|
||||
return sortLocales(locales).map(({ locale }) => ({
|
||||
label: localeLabel(locale),
|
||||
value: locale,
|
||||
}))
|
||||
}
|
||||
finally {
|
||||
eventa.dispose()
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import './amazon-bedrock'
|
||||
import './apple-speech'
|
||||
import './openai'
|
||||
import './openai-audio'
|
||||
import './aihubmix'
|
||||
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
} from '@xsai-ext/providers/utils'
|
||||
import type { ProgressInfo } from '@xsai-transformers/shared/types'
|
||||
import type { MaybePromise } from 'clustr'
|
||||
import type { Component } from 'vue'
|
||||
import type { ComposerTranslation } from 'vue-i18n'
|
||||
import type { $ZodType } from 'zod/v4/core'
|
||||
|
||||
@@ -214,6 +215,12 @@ export interface ProviderDefinition<TConfig extends any = any> {
|
||||
*/
|
||||
configuredBy?: ProviderConfiguredBy
|
||||
|
||||
/** Provider-owned controls for module settings pages. */
|
||||
views?: {
|
||||
/** Lazily loads additional controls shown for this Provider in the Hearing module. */
|
||||
hearing?: () => Promise<{ default: Component }>
|
||||
}
|
||||
|
||||
/** Builds the validation schema and its UI metadata for the current draft. */
|
||||
createProviderConfig: (contextOptions: ProviderConfigContext<TConfig>) => MaybePromise<$ZodType<TConfig>>
|
||||
onboardingFields?: (ctx: { t: ComposerTranslation }) => MaybePromise<ProviderOnboardingField[]>
|
||||
@@ -235,7 +242,7 @@ export interface ProviderDefinition<TConfig extends any = any> {
|
||||
reasoning?: ChatReasoningCapability
|
||||
}
|
||||
transcription?: {
|
||||
protocol: 'websocket' | 'http'
|
||||
protocol: 'websocket' | 'http' | 'native'
|
||||
generateOutput: boolean
|
||||
streamOutput: boolean
|
||||
streamInput: boolean
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
TextStreamer,
|
||||
WhisperForConditionalGeneration,
|
||||
} from '@huggingface/transformers'
|
||||
import { toFloat32FromPCM16 } from '@proj-airi/audio/encoding'
|
||||
import { errorMessageFromValue } from '@proj-airi/stage-shared'
|
||||
|
||||
import { MODEL_IDS, MODEL_NAMES } from '../inference/constants'
|
||||
@@ -162,13 +163,7 @@ async function base64ToFeatures(base64Audio: string): Promise<Float32Array> {
|
||||
bytes[i] = binaryString.charCodeAt(i)
|
||||
}
|
||||
|
||||
const samples = new Int16Array(bytes.buffer.slice(44))
|
||||
const audio = new Float32Array(samples.length)
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
audio[i] = samples[i] / 32768.0
|
||||
}
|
||||
|
||||
return audio
|
||||
return toFloat32FromPCM16(bytes.subarray(44))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { AIRIStreamTranscriptionResult } from '../../libs/providers/stream-
|
||||
import type { StreamingTranscriptionCallbacks, StreamingTranscriptionConsumer } from './streaming-transcription-consumers'
|
||||
|
||||
import { errorMessageFrom, tryCatch } from '@moeru/std'
|
||||
import { toPCM16FromFloat32 } from '@proj-airi/audio/encoding'
|
||||
import { errorMessageFromValue, IOAttributes, IOEvents, IOSpanNames, IOSubsystems } from '@proj-airi/stage-shared'
|
||||
import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables'
|
||||
import { refManualReset } from '@vueuse/core'
|
||||
@@ -21,6 +22,7 @@ import { useAnalytics } from '../../composables/use-analytics'
|
||||
import { activeTurnSpan, startSpan } from '../../composables/use-io-tracer'
|
||||
import { createVadStreamingSession } from '../../libs/audio/vad-streaming-session'
|
||||
import { OFFICIAL_TRANSCRIPTION_PROVIDER_ID } from '../../libs/providers'
|
||||
import { APPLE_SPEECH_TRANSCRIPTION_PROVIDER_ID, executeAppleSpeechStream } from '../../libs/providers/providers/apple-speech'
|
||||
import { streamWebSpeechAPITranscription } from '../../libs/providers/providers/browser-web-speech-api'
|
||||
import { streamTranscription } from '../../libs/providers/stream-transcription'
|
||||
import { useVAD } from '../ai/models/vad'
|
||||
@@ -235,6 +237,7 @@ export function resolveTranscriptionFileName(file: File, explicitFileName?: stri
|
||||
|
||||
const STREAM_TRANSCRIPTION_EXECUTORS: Record<string, StreamTranscription> = {
|
||||
'aliyun-nls-transcription': streamTranscription,
|
||||
[APPLE_SPEECH_TRANSCRIPTION_PROVIDER_ID]: executeAppleSpeechStream,
|
||||
[OFFICIAL_TRANSCRIPTION_PROVIDER_ID]: streamTranscription,
|
||||
// Web Speech API is handled specially in transcribeForMediaStream since it works directly with MediaStream
|
||||
}
|
||||
@@ -796,21 +799,11 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
return await stopRealtimeTranscription(abort, disposeProviderId)
|
||||
}
|
||||
|
||||
function float32ToInt16(buffer: Float32Array) {
|
||||
const output = new Int16Array(buffer.length)
|
||||
for (let i = 0; i < buffer.length; i++) {
|
||||
const value = Math.max(-1, Math.min(1, buffer[i]))
|
||||
output[i] = value < 0 ? value * 0x8000 : value * 0x7FFF
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
function enqueueVadAudio(segment: NonNullable<typeof streamingVadSession.value>['activeSegment'], buffer: Float32Array) {
|
||||
if (!segment)
|
||||
return
|
||||
|
||||
const pcm16 = float32ToInt16(buffer)
|
||||
const pcm16 = toPCM16FromFloat32(buffer)
|
||||
const chunk = pcm16.buffer.slice(0)
|
||||
if (segment.audioStreamController) {
|
||||
segment.audioStreamController.enqueue(chunk)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import type { Page } from 'playwright'
|
||||
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleHearing, configureOnboarding } from '../shared/configurations'
|
||||
import { enableHearingPlaygroundMicrophone, readHearingPlaygroundTranscriptions } from '../shared/interactions'
|
||||
import { appleSpeechAsr } from '../shared/providers'
|
||||
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// Chromium starts the non-looping fake microphone as soon as getUserMedia opens.
|
||||
// Hearing requests microphone permission before the test starts monitoring, so a
|
||||
// short fixture can finish before the Provider receives a speech segment. This
|
||||
// fixture keeps 20 seconds of leading silence before the native transcription.
|
||||
const input = new URL('../long-leading-silence/input.test.wav', import.meta.url)
|
||||
const preflight = [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const isMacOS = await context.runtime.runtimePage.evaluate(() => 'platform' in window && window.platform === 'darwin')
|
||||
context.skip(!isMacOS, 'Apple Speech requires macOS 26 or later.')
|
||||
if (!isMacOS)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: appleSpeechAsr({ locale: 'en-US' }),
|
||||
}
|
||||
}),
|
||||
]
|
||||
|
||||
async function waitForStoredLocale(page: Page, locale: string) {
|
||||
await page.waitForFunction((expectedLocale) => {
|
||||
const stored = localStorage.getItem('settings/providers/configured')
|
||||
if (!stored)
|
||||
return false
|
||||
const providers = JSON.parse(stored) as Record<string, { config?: { locale?: string } }>
|
||||
return providers['apple-speech-transcription']?.config?.locale === expectedLocale
|
||||
}, locale)
|
||||
}
|
||||
|
||||
describe('Apple Speech audio input', () => {
|
||||
it('configures the native locale and transcribes through the Electron Provider', { input, preflight }, async ({ audio }) => {
|
||||
const page = audio.runtimePage
|
||||
audio.activatePage(page)
|
||||
await page.evaluate(() => {
|
||||
window.location.hash = '/settings/modules/hearing'
|
||||
})
|
||||
await page.waitForURL(/#\/settings\/modules\/hearing/)
|
||||
await page.getByTestId('hearing-playground-monitor-toggle').waitFor({ state: 'visible', timeout: 60_000 })
|
||||
const localeCombobox = page.getByTestId('apple-speech-locale').getByRole('combobox')
|
||||
try {
|
||||
await localeCombobox.waitFor({ state: 'visible', timeout: 10_000 })
|
||||
}
|
||||
catch (error) {
|
||||
const diagnostics = await page.evaluate(() => ({
|
||||
activeProvider: localStorage.getItem('settings/hearing/active-provider'),
|
||||
configuredProviders: localStorage.getItem('settings/providers/configured'),
|
||||
localeFieldCount: document.querySelectorAll('[data-testid="apple-speech-locale"]').length,
|
||||
localeTextVisible: document.body.textContent?.includes('Locale') ?? false,
|
||||
}))
|
||||
throw new Error(`Apple Speech locale field is unavailable: ${JSON.stringify(diagnostics)}`, { cause: error })
|
||||
}
|
||||
await localeCombobox.click()
|
||||
const localeOptions = await page.getByRole('option').allTextContents()
|
||||
expect(localeOptions.some(option => option.includes('en-US'))).toBe(true)
|
||||
|
||||
const zhCNOption = page.getByRole('option').filter({ hasText: 'zh-CN' }).first()
|
||||
await zhCNOption.click()
|
||||
await waitForStoredLocale(page, 'zh-CN')
|
||||
|
||||
await localeCombobox.click()
|
||||
const enUSOption = page.getByRole('option').filter({ hasText: 'en-US' }).first()
|
||||
await enUSOption.click()
|
||||
await waitForStoredLocale(page, 'en-US')
|
||||
|
||||
await enableHearingPlaygroundMicrophone(page)
|
||||
try {
|
||||
await readHearingPlaygroundTranscriptions(page, 1)
|
||||
}
|
||||
catch (error) {
|
||||
const diagnostics = await page.evaluate(() => ({
|
||||
activeModel: localStorage.getItem('settings/hearing/active-model'),
|
||||
activeProvider: localStorage.getItem('settings/hearing/active-provider'),
|
||||
configuredProviders: localStorage.getItem('settings/providers/configured'),
|
||||
piniaActionEvents: window.__airiAudioInputE2E?.piniaActionEvents ?? [],
|
||||
probeInstalled: Boolean(window.__airiAudioInputE2E),
|
||||
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
|
||||
streamingTranscriptionUpdates: window.__airiAudioInputE2E?.streamingTranscriptionUpdates ?? [],
|
||||
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
|
||||
}))
|
||||
throw new Error(`Apple Speech did not produce a transcript: ${JSON.stringify(diagnostics)}`, { cause: error })
|
||||
}
|
||||
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Just let go.'],
|
||||
], { match: 'contains' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,19 @@
|
||||
import type { ProviderConfiguration } from '../configurations/provider'
|
||||
|
||||
/** Options for the Apple Speech Provider used by an Electron audio case. */
|
||||
export interface AppleSpeechAsrOptions {
|
||||
/** @default 'en-US' */
|
||||
locale?: string
|
||||
}
|
||||
|
||||
/** Creates the macOS Apple Speech Provider configuration for one Electron case. */
|
||||
export function appleSpeechAsr(options: AppleSpeechAsrOptions = {}): ProviderConfiguration {
|
||||
return {
|
||||
id: 'apple-speech-transcription',
|
||||
definitionId: 'apple-speech-transcription',
|
||||
model: 'apple-speech',
|
||||
config: {
|
||||
locale: options.locale ?? 'en-US',
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
export { aliyunNlsAsr } from './aliyun-nls'
|
||||
export type { AliyunNlsAsrOptions } from './aliyun-nls'
|
||||
export { appleSpeechAsr } from './apple-speech'
|
||||
export type { AppleSpeechAsrOptions } from './apple-speech'
|
||||
export { openaiAsr, openaiLlm, openaiTts } from './openai'
|
||||
export type { OpenAIProviderOptions, OpenAISpeechProviderOptions } from './openai'
|
||||
|
||||
Reference in New Issue
Block a user