feat(stage-*): Aliyun NLS as part of ASR provider
This commit is contained in:
Neko Ayaka
+498
@@ -0,0 +1,498 @@
|
||||
<script setup lang="ts">
|
||||
import type { HearingTranscriptionResult } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
import type { ServerEvent, ServerEvents } from '@proj-airi/stage-ui/stores/providers/aliyun'
|
||||
import type { RemovableRef } from '@vueuse/core'
|
||||
import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/shared-providers'
|
||||
|
||||
import vadWorkletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url'
|
||||
|
||||
import {
|
||||
Alert,
|
||||
Button,
|
||||
ProviderBasicSettings,
|
||||
ProviderSettingsContainer,
|
||||
ProviderSettingsLayout,
|
||||
} from '@proj-airi/stage-ui/components'
|
||||
import { useProviderValidation } from '@proj-airi/stage-ui/composables/use-provider-validation'
|
||||
import { useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
|
||||
import { FieldInput, FieldSelect } from '@proj-airi/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { computed, onBeforeUnmount, reactive, ref, shallowRef } from 'vue'
|
||||
|
||||
const providerId = 'aliyun-nls-transcription'
|
||||
const defaultModel = 'aliyun-nls-v1'
|
||||
const SAMPLE_RATE = 16000
|
||||
|
||||
const regionOptions = [
|
||||
{ label: 'cn-shanghai', value: 'cn-shanghai' },
|
||||
{ label: 'cn-beijing', value: 'cn-beijing' },
|
||||
{ label: 'cn-shenzhen', value: 'cn-shenzhen' },
|
||||
{ label: 'cn-shanghai (internal)', value: 'cn-shanghai-internal' },
|
||||
{ label: 'cn-beijing (internal)', value: 'cn-beijing-internal' },
|
||||
{ label: 'cn-shenzhen (internal)', value: 'cn-shenzhen-internal' },
|
||||
]
|
||||
|
||||
const hearingStore = useHearingStore()
|
||||
const providersStore = useProvidersStore()
|
||||
const { providers } = storeToRefs(providersStore) as { providers: RemovableRef<Record<string, any>> }
|
||||
|
||||
providersStore.initializeProvider(providerId)
|
||||
|
||||
const credentials = reactive({
|
||||
get accessKeyId() {
|
||||
return providers.value[providerId]?.accessKeyId || ''
|
||||
},
|
||||
set accessKeyId(value: string) {
|
||||
ensureProviderCredentials()
|
||||
providers.value[providerId].accessKeyId = value
|
||||
},
|
||||
get accessKeySecret() {
|
||||
return providers.value[providerId]?.accessKeySecret || ''
|
||||
},
|
||||
set accessKeySecret(value: string) {
|
||||
ensureProviderCredentials()
|
||||
providers.value[providerId].accessKeySecret = value
|
||||
},
|
||||
get appKey() {
|
||||
return providers.value[providerId]?.appKey || ''
|
||||
},
|
||||
set appKey(value: string) {
|
||||
ensureProviderCredentials()
|
||||
providers.value[providerId].appKey = value
|
||||
},
|
||||
get region() {
|
||||
return providers.value[providerId]?.region || 'cn-shanghai'
|
||||
},
|
||||
set region(value: string) {
|
||||
ensureProviderCredentials()
|
||||
providers.value[providerId].region = value
|
||||
},
|
||||
})
|
||||
|
||||
function ensureProviderCredentials() {
|
||||
if (!providers.value[providerId]) {
|
||||
providers.value[providerId] = {
|
||||
accessKeyId: '',
|
||||
accessKeySecret: '',
|
||||
appKey: '',
|
||||
region: 'cn-shanghai',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const credentialsReady = computed(() => {
|
||||
return Boolean(
|
||||
credentials.accessKeyId.trim()
|
||||
&& credentials.accessKeySecret.trim()
|
||||
&& credentials.appKey.trim(),
|
||||
)
|
||||
})
|
||||
|
||||
const isRecording = ref(false)
|
||||
const isStreaming = ref(false)
|
||||
const errorMessage = ref<string | null>(null)
|
||||
const currentPartial = ref('')
|
||||
const transcripts = ref<Array<{ index: number, text: string, final: boolean }>>([])
|
||||
|
||||
const audioContext = shallowRef<AudioContext>()
|
||||
const workletNode = shallowRef<AudioWorkletNode>()
|
||||
const mediaStream = shallowRef<MediaStream>()
|
||||
const mediaStreamSource = shallowRef<MediaStreamAudioSourceNode>()
|
||||
const audioStreamController = shallowRef<ReadableStreamDefaultController<ArrayBuffer>>()
|
||||
const transcriptionAbortController = shallowRef<AbortController>()
|
||||
const activeTranscription = shallowRef<HearingTranscriptionResult | null>(null)
|
||||
const transcriptionTextPromise = shallowRef<Promise<string> | null>(null)
|
||||
|
||||
const canStart = computed(() => credentialsReady.value && !isRecording.value && !isStreaming.value)
|
||||
const canStop = computed(() => isRecording.value || isStreaming.value)
|
||||
const canAbort = computed(() => isStreaming.value && Boolean(transcriptionAbortController.value))
|
||||
|
||||
const {
|
||||
t,
|
||||
router,
|
||||
providerMetadata,
|
||||
isValidating,
|
||||
isValid,
|
||||
validationMessage,
|
||||
handleResetSettings,
|
||||
} = useProviderValidation(providerId)
|
||||
|
||||
function float32ToInt16(buffer: Float32Array) {
|
||||
const output = new Int16Array(buffer.length)
|
||||
for (let i = 0; i < buffer.length; i++) {
|
||||
const value = Math.max(-1, Math.min(1, buffer[i]))
|
||||
output[i] = value < 0 ? value * 0x8000 : value * 0x7FFF
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
async function initializeAudioGraph(stream: MediaStream) {
|
||||
const context = new AudioContext({
|
||||
sampleRate: SAMPLE_RATE,
|
||||
latencyHint: 'interactive',
|
||||
})
|
||||
await context.audioWorklet.addModule(vadWorkletUrl)
|
||||
|
||||
const node = new AudioWorkletNode(context, 'vad-audio-worklet-processor')
|
||||
node.port.onmessage = ({ data }: MessageEvent<{ buffer?: Float32Array }>) => {
|
||||
const buffer = data.buffer
|
||||
const controller = audioStreamController.value
|
||||
if (!buffer || !controller)
|
||||
return
|
||||
|
||||
const pcm16 = float32ToInt16(buffer)
|
||||
controller.enqueue(pcm16.buffer.slice(0))
|
||||
}
|
||||
|
||||
const source = context.createMediaStreamSource(stream)
|
||||
source.connect(node)
|
||||
|
||||
const silentGain = context.createGain()
|
||||
silentGain.gain.value = 0
|
||||
node.connect(silentGain)
|
||||
silentGain.connect(context.destination)
|
||||
|
||||
audioContext.value = context
|
||||
workletNode.value = node
|
||||
mediaStreamSource.value = source
|
||||
}
|
||||
|
||||
function resetTranscriptionOutput() {
|
||||
currentPartial.value = ''
|
||||
transcripts.value = []
|
||||
}
|
||||
|
||||
function handleServerEvent(event: ServerEvent) {
|
||||
switch (event.header.name) {
|
||||
case 'TranscriptionResultChanged': {
|
||||
const payload = event.payload as ServerEvents['TranscriptionResultChanged']
|
||||
currentPartial.value = payload.result
|
||||
upsertTranscript(payload.index, payload.result, false)
|
||||
break
|
||||
}
|
||||
case 'SentenceEnd': {
|
||||
const payload = event.payload as ServerEvents['SentenceEnd']
|
||||
currentPartial.value = ''
|
||||
upsertTranscript(payload.index, payload.result, true)
|
||||
break
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
function upsertTranscript(index: number, text: string, final: boolean) {
|
||||
const existingIndex = transcripts.value.findIndex(entry => entry.index === index)
|
||||
|
||||
if (existingIndex >= 0) {
|
||||
const existing = transcripts.value[existingIndex]
|
||||
transcripts.value.splice(existingIndex, 1, {
|
||||
index,
|
||||
text,
|
||||
final: existing.final || final,
|
||||
})
|
||||
}
|
||||
else {
|
||||
transcripts.value.push({ index, text, final })
|
||||
}
|
||||
|
||||
transcripts.value.sort((a, b) => a.index - b.index)
|
||||
}
|
||||
|
||||
async function startStreaming() {
|
||||
if (!canStart.value)
|
||||
return
|
||||
|
||||
errorMessage.value = null
|
||||
resetTranscriptionOutput()
|
||||
|
||||
const abortController = new AbortController()
|
||||
transcriptionAbortController.value = abortController
|
||||
|
||||
const audioStream = new ReadableStream<ArrayBuffer>({
|
||||
start(controller) {
|
||||
audioStreamController.value = controller
|
||||
},
|
||||
cancel: () => {
|
||||
audioStreamController.value = undefined
|
||||
},
|
||||
})
|
||||
|
||||
try {
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
|
||||
if (!provider)
|
||||
throw new Error('Failed to initialize Aliyun NLS provider.')
|
||||
|
||||
const result = await hearingStore.transcription(
|
||||
providerId,
|
||||
provider,
|
||||
defaultModel,
|
||||
{ inputAudioStream: audioStream },
|
||||
undefined,
|
||||
{
|
||||
providerOptions: {
|
||||
abortSignal: abortController.signal,
|
||||
hooks: {
|
||||
onServerEvent: (event: ServerEvent) => {
|
||||
handleServerEvent(event)
|
||||
},
|
||||
},
|
||||
onSessionTerminated: async (error?: unknown) => {
|
||||
if (error)
|
||||
errorMessage.value = error instanceof Error ? error.message : String(error)
|
||||
isStreaming.value = false
|
||||
transcriptionAbortController.value = undefined
|
||||
},
|
||||
sessionOptions: {
|
||||
format: 'pcm',
|
||||
sample_rate: SAMPLE_RATE,
|
||||
enable_punctuation_prediction: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
if (result.mode !== 'stream')
|
||||
throw new Error('Aliyun NLS returned a non-streaming result unexpectedly.')
|
||||
|
||||
activeTranscription.value = result
|
||||
transcriptionTextPromise.value = result.text
|
||||
.catch((error) => {
|
||||
errorMessage.value = error instanceof Error ? error.message : String(error)
|
||||
throw error
|
||||
})
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
sampleRate: SAMPLE_RATE,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
})
|
||||
|
||||
mediaStream.value = stream
|
||||
await initializeAudioGraph(stream)
|
||||
|
||||
if (audioContext.value?.state === 'suspended')
|
||||
await audioContext.value.resume()
|
||||
|
||||
isRecording.value = true
|
||||
isStreaming.value = true
|
||||
}
|
||||
catch (error) {
|
||||
errorMessage.value = error instanceof Error ? error.message : String(error)
|
||||
await stopStreaming()
|
||||
}
|
||||
}
|
||||
|
||||
async function stopStreaming() {
|
||||
try {
|
||||
workletNode.value?.port.postMessage({ type: 'stop' })
|
||||
}
|
||||
catch { /* noop */ }
|
||||
|
||||
if (mediaStreamSource.value) {
|
||||
mediaStreamSource.value.disconnect()
|
||||
mediaStreamSource.value = undefined
|
||||
}
|
||||
|
||||
if (workletNode.value) {
|
||||
workletNode.value.port.onmessage = null
|
||||
workletNode.value.disconnect()
|
||||
workletNode.value = undefined
|
||||
}
|
||||
|
||||
if (mediaStream.value) {
|
||||
mediaStream.value.getTracks().forEach(track => track.stop())
|
||||
mediaStream.value = undefined
|
||||
}
|
||||
|
||||
if (audioContext.value) {
|
||||
try {
|
||||
await audioContext.value.close()
|
||||
}
|
||||
catch { /* noop */ }
|
||||
audioContext.value = undefined
|
||||
}
|
||||
|
||||
audioStreamController.value?.close()
|
||||
audioStreamController.value = undefined
|
||||
|
||||
isRecording.value = false
|
||||
|
||||
if (transcriptionTextPromise.value) {
|
||||
try {
|
||||
await transcriptionTextPromise.value
|
||||
}
|
||||
catch { /* handled in promise */ }
|
||||
finally {
|
||||
transcriptionTextPromise.value = null
|
||||
}
|
||||
}
|
||||
|
||||
isStreaming.value = false
|
||||
transcriptionAbortController.value = undefined
|
||||
activeTranscription.value = null
|
||||
}
|
||||
|
||||
function abortStreaming() {
|
||||
const controller = transcriptionAbortController.value
|
||||
if (!controller)
|
||||
return
|
||||
|
||||
controller.abort(new DOMException('Aborted by user', 'AbortError'))
|
||||
audioStreamController.value?.error(new DOMException('Aborted by user', 'AbortError'))
|
||||
audioStreamController.value = undefined
|
||||
void stopStreaming()
|
||||
}
|
||||
|
||||
onBeforeUnmount(async () => {
|
||||
abortStreaming()
|
||||
await stopStreaming()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<ProviderSettingsLayout
|
||||
:provider-name="providerMetadata?.localizedName"
|
||||
:provider-icon="providerMetadata?.icon"
|
||||
:provider-icon-color="providerMetadata?.iconColor"
|
||||
:on-back="() => router.back()"
|
||||
>
|
||||
<div class="flex flex-col gap-6 md:flex-row">
|
||||
<ProviderSettingsContainer class="w-full md:w-[40%] space-y-6">
|
||||
<ProviderBasicSettings
|
||||
:title="t('settings.pages.providers.common.section.basic.title')"
|
||||
:description="t('settings.pages.providers.common.section.basic.description')"
|
||||
:on-reset="handleResetSettings"
|
||||
>
|
||||
<FieldInput
|
||||
v-model="credentials.accessKeyId"
|
||||
label="Access Key ID"
|
||||
placeholder="LTAI..."
|
||||
/>
|
||||
|
||||
<FieldInput
|
||||
v-model="credentials.accessKeySecret"
|
||||
label="Access Key Secret"
|
||||
type="password"
|
||||
placeholder="****************"
|
||||
/>
|
||||
|
||||
<FieldInput
|
||||
v-model="credentials.appKey"
|
||||
label="App Key"
|
||||
placeholder="请输入 AppKey"
|
||||
/>
|
||||
|
||||
<FieldSelect
|
||||
v-model="credentials.region"
|
||||
label="Region"
|
||||
:options="regionOptions"
|
||||
layout="vertical"
|
||||
/>
|
||||
</ProviderBasicSettings>
|
||||
|
||||
<Alert v-if="!isValid && isValidating === 0 && validationMessage" type="error">
|
||||
<template #title>
|
||||
{{ t('settings.dialogs.onboarding.validationFailed') }}
|
||||
</template>
|
||||
<template #content>
|
||||
<div class="whitespace-pre-wrap break-all">
|
||||
{{ validationMessage }}
|
||||
</div>
|
||||
</template>
|
||||
</Alert>
|
||||
|
||||
<Alert v-if="isValid && isValidating === 0" type="success">
|
||||
<template #title>
|
||||
{{ t('settings.dialogs.onboarding.validationSuccess') }}
|
||||
</template>
|
||||
</Alert>
|
||||
</ProviderSettingsContainer>
|
||||
|
||||
<div class="w-full flex flex-1 flex-col gap-6">
|
||||
<div class="border border-neutral-200/80 rounded-xl bg-neutral-50/60 p-4 dark:border-neutral-700 dark:bg-neutral-900/40">
|
||||
<div class="flex flex-wrap items-center justify-between gap-3">
|
||||
<div class="space-x-3">
|
||||
<Button :disabled="!canStart" variant="primary" @click="startStreaming">
|
||||
{{ isRecording ? 'Streaming...' : 'Start Realtime Transcription' }}
|
||||
</Button>
|
||||
<Button :disabled="!canStop" variant="secondary" @click="stopStreaming">
|
||||
Stop
|
||||
</Button>
|
||||
<Button
|
||||
v-if="isStreaming"
|
||||
:disabled="!canAbort"
|
||||
@click="abortStreaming"
|
||||
>
|
||||
Abort Session
|
||||
</Button>
|
||||
</div>
|
||||
<div class="text-sm text-neutral-500 dark:text-neutral-400">
|
||||
<span v-if="isRecording" class="rounded bg-red-500/10 px-2 py-0.5 text-xs text-red-500">
|
||||
Recording
|
||||
</span>
|
||||
<span v-else-if="isStreaming" class="rounded bg-blue-500/10 px-2 py-0.5 text-xs text-blue-500">
|
||||
Connected
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p v-if="errorMessage" class="mt-3 text-sm text-red-500">
|
||||
{{ errorMessage }}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="border border-neutral-200/80 rounded-xl bg-neutral-50/60 p-4 dark:border-neutral-700 dark:bg-neutral-900/40">
|
||||
<h2 class="text-lg font-semibold">
|
||||
Transcripts
|
||||
</h2>
|
||||
<div v-if="currentPartial" class="mt-3 text-sm text-neutral-500 dark:text-neutral-400">
|
||||
<div class="text-xs text-neutral-400 tracking-wide uppercase dark:text-neutral-500">
|
||||
Partial
|
||||
</div>
|
||||
<div class="mt-1 font-medium">
|
||||
{{ currentPartial }}
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="!transcripts.length && !currentPartial" class="mt-3 text-sm text-neutral-400 dark:text-neutral-600">
|
||||
Waiting for audio...
|
||||
</div>
|
||||
<ul class="mt-4 text-sm space-y-3">
|
||||
<li
|
||||
v-for="sentence in transcripts"
|
||||
:key="sentence.index"
|
||||
class="flex items-start gap-3"
|
||||
>
|
||||
<span class="mt-0.5 rounded bg-neutral-200/80 px-2 py-0.5 text-xs text-neutral-700 dark:bg-neutral-800/70 dark:text-neutral-200">
|
||||
#{{ sentence.index }}
|
||||
</span>
|
||||
<div>
|
||||
<div
|
||||
class="font-medium"
|
||||
:class="sentence.final ? '' : 'italic text-neutral-500 dark:text-neutral-400'"
|
||||
>
|
||||
{{ sentence.text }}
|
||||
</div>
|
||||
<div v-if="!sentence.final" class="text-xs text-neutral-400">
|
||||
Awaiting final result...
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ProviderSettingsLayout>
|
||||
</template>
|
||||
|
||||
<route lang="yaml">
|
||||
meta:
|
||||
layout: settings
|
||||
stageTransition:
|
||||
name: slide
|
||||
</route>
|
||||
+3
-2
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import type { RemovableRef } from '@vueuse/core'
|
||||
import type { TranscriptionProvider } from '@xsai-ext/shared-providers'
|
||||
import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/shared-providers'
|
||||
|
||||
import {
|
||||
Alert,
|
||||
@@ -57,11 +57,12 @@ const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
|
||||
|
||||
// Generate transcription
|
||||
async function handleGenerateTranscription(file: File) {
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProvider<string>>(providerId)
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
|
||||
if (!provider)
|
||||
throw new Error('Failed to initialize transcription provider')
|
||||
|
||||
return await hearingStore.transcription(
|
||||
providerId,
|
||||
provider,
|
||||
model.value,
|
||||
file,
|
||||
|
||||
+3
-2
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import type { TranscriptionProvider } from '@xsai-ext/shared-providers'
|
||||
import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/shared-providers'
|
||||
|
||||
import {
|
||||
TranscriptionPlayground,
|
||||
@@ -23,7 +23,7 @@ const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
|
||||
|
||||
// Generate speech with ElevenLabs-specific parameters
|
||||
async function handleGenerateTranscription(file: File) {
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProvider<string>>(providerId)
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
|
||||
if (!provider) {
|
||||
throw new Error('Failed to initialize speech provider')
|
||||
}
|
||||
@@ -36,6 +36,7 @@ async function handleGenerateTranscription(file: File) {
|
||||
|
||||
// ElevenLabs doesn't need SSML conversion, but if SSML is provided, use it directly
|
||||
return await hearingStore.transcription(
|
||||
providerId,
|
||||
provider,
|
||||
model,
|
||||
file,
|
||||
|
||||
+3
-2
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import type { RemovableRef } from '@vueuse/core'
|
||||
import type { TranscriptionProvider } from '@xsai-ext/shared-providers'
|
||||
import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/shared-providers'
|
||||
|
||||
import {
|
||||
Alert,
|
||||
@@ -57,11 +57,12 @@ const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
|
||||
|
||||
// Generate transcription
|
||||
async function handleGenerateTranscription(file: File) {
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProvider<string>>(providerId)
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
|
||||
if (!provider)
|
||||
throw new Error('Failed to initialize transcription provider')
|
||||
|
||||
return await hearingStore.transcription(
|
||||
providerId,
|
||||
provider,
|
||||
model.value,
|
||||
file,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import type { GenerateTranscriptionResult } from '@xsai/generate-transcription'
|
||||
import type { HearingTranscriptionResult } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
|
||||
import { FieldRange, FieldSelect } from '@proj-airi/ui'
|
||||
import { until } from '@vueuse/core'
|
||||
@@ -14,7 +14,7 @@ import { Button } from '../../misc'
|
||||
|
||||
const props = defineProps<{
|
||||
// Provider-specific handlers (provided from parent)
|
||||
generateTranscription: (input: File) => Promise<GenerateTranscriptionResult<'json' | 'verbose_json', undefined>>
|
||||
generateTranscription: (input: File) => Promise<HearingTranscriptionResult>
|
||||
// Current state
|
||||
apiKeyConfigured?: boolean
|
||||
}>()
|
||||
@@ -107,8 +107,11 @@ onStopRecord(async (recording) => {
|
||||
try {
|
||||
if (recording && recording.size > 0) {
|
||||
audios.value.push(recording)
|
||||
const res = await props.generateTranscription(new File([recording], 'recording.wav'))
|
||||
transcriptions.value.push(res.text)
|
||||
const result = await props.generateTranscription(new File([recording], 'recording.wav'))
|
||||
const text = result.mode === 'stream'
|
||||
? await result.text
|
||||
: result.text
|
||||
transcriptions.value.push(text)
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import type { TranscriptionProvider, TranscriptionProviderWithExtraOptions } from '@xsai-ext/shared-providers'
|
||||
import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/shared-providers'
|
||||
|
||||
import type { StreamTranscriptionResult } from '../providers/aliyun'
|
||||
|
||||
import { useLocalStorage } from '@vueuse/core'
|
||||
import { generateTranscription } from '@xsai/generate-transcription'
|
||||
@@ -6,6 +8,25 @@ import { defineStore, storeToRefs } from 'pinia'
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useProvidersStore } from '../providers'
|
||||
import { streamTranscription as streamAliyunTranscription } from '../providers/aliyun'
|
||||
|
||||
type GenerateTranscriptionResponse = Awaited<ReturnType<typeof generateTranscription>>
|
||||
type HearingTranscriptionGenerateResult = GenerateTranscriptionResponse & { mode: 'generate' }
|
||||
type HearingTranscriptionStreamResult = StreamTranscriptionResult & { mode: 'stream' }
|
||||
export type HearingTranscriptionResult = HearingTranscriptionGenerateResult | HearingTranscriptionStreamResult
|
||||
|
||||
type HearingTranscriptionInput = File | {
|
||||
file?: File
|
||||
inputAudioStream?: ReadableStream<ArrayBuffer>
|
||||
}
|
||||
|
||||
interface HearingTranscriptionInvokeOptions {
|
||||
providerOptions?: Record<string, unknown>
|
||||
}
|
||||
|
||||
const STREAM_TRANSCRIPTION_EXECUTORS: Record<string, typeof streamAliyunTranscription> = {
|
||||
'aliyun-nls-transcription': streamAliyunTranscription,
|
||||
}
|
||||
|
||||
export const useHearingStore = defineStore('hearing-store', () => {
|
||||
const providersStore = useProvidersStore()
|
||||
@@ -56,18 +77,78 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
})
|
||||
|
||||
async function transcription(
|
||||
providerId: string,
|
||||
provider: TranscriptionProviderWithExtraOptions<string, any>,
|
||||
model: string,
|
||||
file: File,
|
||||
input: HearingTranscriptionInput,
|
||||
format?: 'json' | 'verbose_json',
|
||||
) {
|
||||
options?: HearingTranscriptionInvokeOptions,
|
||||
): Promise<HearingTranscriptionResult> {
|
||||
const normalizedInput = (input instanceof File ? { file: input } : input ?? {}) as {
|
||||
file?: File
|
||||
inputAudioStream?: ReadableStream<ArrayBuffer>
|
||||
}
|
||||
const features = providersStore.getTranscriptionFeatures(providerId)
|
||||
const streamExecutor = STREAM_TRANSCRIPTION_EXECUTORS[providerId]
|
||||
|
||||
if (features.supportsStreamOutput && streamExecutor) {
|
||||
const request = provider.transcription(model, options?.providerOptions)
|
||||
|
||||
if (features.supportsStreamInput && normalizedInput.inputAudioStream) {
|
||||
const streamResult = streamExecutor({
|
||||
...request,
|
||||
inputAudioStream: normalizedInput.inputAudioStream,
|
||||
} as Parameters<typeof streamExecutor>[0])
|
||||
// TODO: integrate VAD-driven silence detection to stop and restart realtime sessions based on silence thresholds.
|
||||
return {
|
||||
mode: 'stream',
|
||||
...streamResult,
|
||||
}
|
||||
}
|
||||
|
||||
if (!features.supportsStreamInput && normalizedInput.file) {
|
||||
const streamResult = streamExecutor({
|
||||
...request,
|
||||
file: normalizedInput.file,
|
||||
} as Parameters<typeof streamExecutor>[0])
|
||||
// TODO: integrate VAD-driven silence detection to stop and restart realtime sessions based on silence thresholds.
|
||||
return {
|
||||
mode: 'stream',
|
||||
...streamResult,
|
||||
}
|
||||
}
|
||||
|
||||
if (features.supportsStreamInput && !normalizedInput.inputAudioStream && normalizedInput.file) {
|
||||
const streamResult = streamExecutor({
|
||||
...request,
|
||||
file: normalizedInput.file,
|
||||
} as Parameters<typeof streamExecutor>[0])
|
||||
// TODO: integrate VAD-driven silence detection to stop and restart realtime sessions based on silence thresholds.
|
||||
return {
|
||||
mode: 'stream',
|
||||
...streamResult,
|
||||
}
|
||||
}
|
||||
|
||||
if (!features.supportsGenerate || !normalizedInput.file) {
|
||||
throw new Error('No compatible input provided for streaming transcription.')
|
||||
}
|
||||
}
|
||||
|
||||
if (!normalizedInput.file) {
|
||||
throw new Error('File input is required for transcription.')
|
||||
}
|
||||
|
||||
const response = await generateTranscription({
|
||||
...provider.transcription(model),
|
||||
file,
|
||||
...provider.transcription(model, options?.providerOptions),
|
||||
file: normalizedInput.file,
|
||||
responseFormat: format,
|
||||
})
|
||||
|
||||
return response
|
||||
return {
|
||||
mode: 'generate',
|
||||
...response,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -102,15 +183,21 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
|
||||
try {
|
||||
if (recording && recording.size > 0) {
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProvider<string>>(activeTranscriptionProvider.value)
|
||||
const providerId = activeTranscriptionProvider.value
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
|
||||
if (!provider) {
|
||||
throw new Error('Failed to initialize speech provider')
|
||||
}
|
||||
|
||||
// Get model from configuration or use default
|
||||
const model = activeTranscriptionModel.value
|
||||
const res = await hearingStore.transcription(provider, model, new File([recording], 'recording.wav'))
|
||||
return res.text
|
||||
const result = await hearingStore.transcription(
|
||||
providerId,
|
||||
provider,
|
||||
model,
|
||||
new File([recording], 'recording.wav'),
|
||||
)
|
||||
return result.mode === 'stream' ? await result.text : result.text
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
|
||||
@@ -60,6 +60,19 @@ import { useI18n } from 'vue-i18n'
|
||||
|
||||
import { models as elevenLabsModels } from './providers/elevenlabs/list-models'
|
||||
import { buildOpenAICompatibleProvider } from './providers/openai-compatible-builder'
|
||||
import type { AliyunRealtimeSpeechExtraOptions } from './providers/aliyun/stream-transcription'
|
||||
import { createAliyunNLSProvider as createAliyunNlsStreamProvider } from './providers/aliyun/stream-transcription'
|
||||
|
||||
const ALIYUN_NLS_REGIONS = [
|
||||
'cn-shanghai',
|
||||
'cn-shanghai-internal',
|
||||
'cn-beijing',
|
||||
'cn-beijing-internal',
|
||||
'cn-shenzhen',
|
||||
'cn-shenzhen-internal',
|
||||
] as const
|
||||
|
||||
type AliyunNlsRegion = typeof ALIYUN_NLS_REGIONS[number]
|
||||
|
||||
export interface ProviderMetadata {
|
||||
id: string
|
||||
@@ -141,6 +154,11 @@ export interface ProviderMetadata {
|
||||
valid: boolean
|
||||
}
|
||||
}
|
||||
transcriptionFeatures?: {
|
||||
supportsGenerate: boolean
|
||||
supportsStreamOutput: boolean
|
||||
supportsStreamInput: boolean
|
||||
}
|
||||
}
|
||||
|
||||
export interface ModelInfo {
|
||||
@@ -841,6 +859,87 @@ export const useProvidersStore = defineStore('providers', () => {
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
|
||||
creator: createOpenAI,
|
||||
}),
|
||||
'aliyun-nls-transcription': {
|
||||
id: 'aliyun-nls-transcription',
|
||||
category: 'transcription',
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'],
|
||||
nameKey: 'settings.pages.providers.provider.aliyun-nls.title',
|
||||
name: 'Aliyun NLS',
|
||||
descriptionKey: 'settings.pages.providers.provider.aliyun-nls.description',
|
||||
description: 'nls-console.aliyun.com',
|
||||
icon: 'i-lobe-icons:alibabacloud',
|
||||
defaultOptions: () => ({
|
||||
accessKeyId: '',
|
||||
accessKeySecret: '',
|
||||
appKey: '',
|
||||
region: 'cn-shanghai',
|
||||
}),
|
||||
transcriptionFeatures: {
|
||||
supportsGenerate: false,
|
||||
supportsStreamOutput: true,
|
||||
supportsStreamInput: true,
|
||||
},
|
||||
createProvider: async (config) => {
|
||||
const toString = (value: unknown) => typeof value === 'string' ? value.trim() : ''
|
||||
|
||||
const accessKeyId = toString(config.accessKeyId)
|
||||
const accessKeySecret = toString(config.accessKeySecret)
|
||||
const appKey = toString(config.appKey)
|
||||
const region = toString(config.region)
|
||||
const resolvedRegion = ALIYUN_NLS_REGIONS.includes(region as AliyunNlsRegion) ? region as AliyunNlsRegion : 'cn-shanghai'
|
||||
|
||||
if (!accessKeyId || !accessKeySecret || !appKey)
|
||||
throw new Error('Aliyun NLS credentials are incomplete.')
|
||||
|
||||
const provider = createAliyunNlsStreamProvider(accessKeyId, accessKeySecret, appKey, { region: resolvedRegion })
|
||||
|
||||
return {
|
||||
transcription(model: string, extraOptions?: AliyunRealtimeSpeechExtraOptions) {
|
||||
return provider.speech(model, extraOptions)
|
||||
},
|
||||
} as TranscriptionProviderWithExtraOptions<string, AliyunRealtimeSpeechExtraOptions>
|
||||
},
|
||||
capabilities: {
|
||||
listModels: async () => {
|
||||
return [
|
||||
{
|
||||
id: 'aliyun-nls-v1',
|
||||
name: 'Aliyun NLS Realtime',
|
||||
provider: 'aliyun-nls-transcription',
|
||||
description: 'Realtime streaming transcription using Aliyun NLS.',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
validators: {
|
||||
validateProviderConfig: (config) => {
|
||||
const errors: Error[] = []
|
||||
const toString = (value: unknown) => typeof value === 'string' ? value.trim() : ''
|
||||
|
||||
const accessKeyId = toString(config.accessKeyId)
|
||||
const accessKeySecret = toString(config.accessKeySecret)
|
||||
const appKey = toString(config.appKey)
|
||||
const region = toString(config.region)
|
||||
|
||||
if (!accessKeyId)
|
||||
errors.push(new Error('Access Key ID is required.'))
|
||||
if (!accessKeySecret)
|
||||
errors.push(new Error('Access Key Secret is required.'))
|
||||
if (!appKey)
|
||||
errors.push(new Error('App Key is required.'))
|
||||
if (region && !ALIYUN_NLS_REGIONS.includes(region as AliyunNlsRegion))
|
||||
errors.push(new Error('Region is invalid.'))
|
||||
|
||||
return {
|
||||
errors,
|
||||
reason: errors.length > 0 ? errors.map(error => error.message).join(', ') : '',
|
||||
valid: errors.length === 0,
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
'anthropic': buildOpenAICompatibleProvider({
|
||||
id: 'anthropic',
|
||||
name: 'Anthropic',
|
||||
@@ -1775,7 +1874,8 @@ export const useProvidersStore = defineStore('providers', () => {
|
||||
const metadata = providerMetadata[providerId]
|
||||
const defaultOptions = metadata.defaultOptions?.() || {}
|
||||
providerCredentials.value[providerId] = {
|
||||
baseUrl: defaultOptions.baseUrl || '',
|
||||
...defaultOptions,
|
||||
...(Object.prototype.hasOwnProperty.call(defaultOptions, 'baseUrl') ? {} : { baseUrl: '' }),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1908,6 +2008,17 @@ export const useProvidersStore = defineStore('providers', () => {
|
||||
}))
|
||||
})
|
||||
|
||||
function getTranscriptionFeatures(providerId: string) {
|
||||
const metadata = providerMetadata[providerId]
|
||||
const features = metadata?.transcriptionFeatures
|
||||
|
||||
return {
|
||||
supportsGenerate: features?.supportsGenerate ?? true,
|
||||
supportsStreamOutput: features?.supportsStreamOutput ?? false,
|
||||
supportsStreamInput: features?.supportsStreamInput ?? false,
|
||||
}
|
||||
}
|
||||
|
||||
// Function to get provider object by provider id
|
||||
async function getProviderInstance<R extends
|
||||
| ChatProvider
|
||||
@@ -1987,6 +2098,7 @@ export const useProvidersStore = defineStore('providers', () => {
|
||||
configuredProviders,
|
||||
providerMetadata,
|
||||
getProviderMetadata,
|
||||
getTranscriptionFeatures,
|
||||
allProvidersMetadata,
|
||||
initializeProvider,
|
||||
validateProvider,
|
||||
|
||||
@@ -107,11 +107,6 @@ async function startRealtimeSession(options: InternalRealtimeOptions): Promise<v
|
||||
abortSignal && abortHandler.on()
|
||||
|
||||
async function cleanup(error?: unknown) {
|
||||
if (closed)
|
||||
return
|
||||
|
||||
closed = true
|
||||
|
||||
abortHandler && abortSignal && abortHandler.off()
|
||||
mayThrow(async () => await reader.cancel())
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ export function buildOpenAICompatibleProvider(
|
||||
validators?: ProviderMetadata['validators']
|
||||
validation?: ('health' | 'model_list' | 'chat_completions')[]
|
||||
additionalHeaders?: Record<string, string>
|
||||
transcriptionFeatures?: ProviderMetadata['transcriptionFeatures']
|
||||
},
|
||||
): ProviderMetadata {
|
||||
const {
|
||||
@@ -66,6 +67,7 @@ export function buildOpenAICompatibleProvider(
|
||||
validators,
|
||||
validation,
|
||||
additionalHeaders,
|
||||
transcriptionFeatures,
|
||||
...rest
|
||||
} = options
|
||||
|
||||
@@ -251,9 +253,11 @@ export function buildOpenAICompatibleProvider(
|
||||
},
|
||||
}
|
||||
|
||||
const resolvedCategory = category ?? 'chat'
|
||||
|
||||
return {
|
||||
id,
|
||||
category: category || 'chat',
|
||||
category: resolvedCategory,
|
||||
tasks: tasks || ['text-generation'],
|
||||
nameKey,
|
||||
name,
|
||||
@@ -270,6 +274,15 @@ export function buildOpenAICompatibleProvider(
|
||||
},
|
||||
capabilities: finalCapabilities,
|
||||
validators: finalValidators,
|
||||
...(resolvedCategory === 'transcription'
|
||||
? {
|
||||
transcriptionFeatures: transcriptionFeatures ?? {
|
||||
supportsGenerate: true,
|
||||
supportsStreamOutput: false,
|
||||
supportsStreamInput: false,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...rest,
|
||||
} as ProviderMetadata
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user