From d6d3987473cc73adb0c1e5864f6b7e26c4b95cac Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Thu, 19 Jun 2025 01:26:42 +0800 Subject: [PATCH] chore(stage-ui,stage-app,stage-tamagotchi): prepare for pure local STT/TTS --- .../providers/openai-audio-speech.vue | 111 ++++++ .../providers/openai-audio-transcription.vue | 111 ++++++ .../providers/openai-audio-speech.vue | 111 ++++++ .../providers/openai-audio-transcription.vue | 111 ++++++ cspell.config.yaml | 1 + packages/stage-ui/package.json | 2 + packages/stage-ui/src/stores/providers.ts | 353 ++++++++++++++++-- pnpm-lock.yaml | 6 + 8 files changed, 769 insertions(+), 37 deletions(-) create mode 100644 apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-speech.vue create mode 100644 apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-transcription.vue create mode 100644 apps/stage-web/src/pages/settings/providers/openai-audio-speech.vue create mode 100644 apps/stage-web/src/pages/settings/providers/openai-audio-transcription.vue diff --git a/apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-speech.vue b/apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-speech.vue new file mode 100644 index 000000000..28bb12c27 --- /dev/null +++ b/apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-speech.vue @@ -0,0 +1,111 @@ + + + + + + meta: + layout: settings + stageTransition: + name: slide + diff --git a/apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-transcription.vue b/apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-transcription.vue new file mode 100644 index 000000000..28bb12c27 --- /dev/null +++ b/apps/stage-tamagotchi/src/pages/settings/providers/openai-audio-transcription.vue @@ -0,0 +1,111 @@ + + + + + + meta: + layout: settings + stageTransition: + name: slide + diff --git a/apps/stage-web/src/pages/settings/providers/openai-audio-speech.vue b/apps/stage-web/src/pages/settings/providers/openai-audio-speech.vue new file mode 100644 index 000000000..28bb12c27 --- /dev/null +++ b/apps/stage-web/src/pages/settings/providers/openai-audio-speech.vue @@ -0,0 +1,111 @@ + + + + + + meta: + layout: settings + stageTransition: + name: slide + diff --git a/apps/stage-web/src/pages/settings/providers/openai-audio-transcription.vue b/apps/stage-web/src/pages/settings/providers/openai-audio-transcription.vue new file mode 100644 index 000000000..28bb12c27 --- /dev/null +++ b/apps/stage-web/src/pages/settings/providers/openai-audio-transcription.vue @@ -0,0 +1,111 @@ + + + + + + meta: + layout: settings + stageTransition: + name: slide + diff --git a/cspell.config.yaml b/cspell.config.yaml index 30243f843..28f3b7eee 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -88,6 +88,7 @@ words: - iconify - intlify - Kawaii + - kebabcase - kwaa - lemonnekogh - libsodium diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index 226ad51f8..c26916c40 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -112,6 +112,7 @@ "@xsai-ext/providers-local": "catalog:", "@xsai-ext/shared-providers": "catalog:", "@xsai-transformers/embed": "^0.0.6", + "@xsai-transformers/shared": "^0.0.6", "@xsai/embed": "catalog:", "@xsai/generate-speech": "catalog:", "@xsai/model": "catalog:", @@ -120,6 +121,7 @@ "@xsai/tool": "catalog:", "culori": "^4.0.1", "defu": "^6.1.4", + "gpuu": "^1.0.3", "jszip": "^3.10.1", "localforage": "^1.10.0", "pinia": "^3.0.3", diff --git a/packages/stage-ui/src/stores/providers.ts b/packages/stage-ui/src/stores/providers.ts index 6ae09f1eb..ebc7095cf 100644 --- a/packages/stage-ui/src/stores/providers.ts +++ b/packages/stage-ui/src/stores/providers.ts @@ -1,4 +1,5 @@ import type { ChatProvider, ChatProviderWithExtraOptions, EmbedProvider, EmbedProviderWithExtraOptions, SpeechProvider, SpeechProviderWithExtraOptions, TranscriptionProvider, TranscriptionProviderWithExtraOptions } from '@xsai-ext/shared-providers' +import type { ProgressInfo } from '@xsai-transformers/shared/types' import type { UnAlibabaCloudOptions, UnElevenLabsOptions, @@ -7,7 +8,7 @@ import type { VoiceProviderWithExtraOptions, } from 'unspeech' -import { useLocalStorage } from '@vueuse/core' +import { computedAsync, useLocalStorage } from '@vueuse/core' import { createAnthropic, createDeepSeek, @@ -25,6 +26,7 @@ import { } from '@xsai-ext/providers-cloud' import { createOllama, createPlayer2 } from '@xsai-ext/providers-local' import { listModels } from '@xsai/model' +import { isWebGPUSupported } from 'gpuu/webgpu' import { defineStore } from 'pinia' import { createUnAlibabaCloud, @@ -40,16 +42,38 @@ import { models as elevenLabsModels } from './providers/elevenlabs/list-models' export interface ProviderMetadata { id: string + order?: number category: 'chat' | 'embed' | 'speech' | 'transcription' tasks: string[] nameKey: string // i18n key for provider name name: string // Default name (fallback) descriptionKey: string // i18n key for description description: string // Default description (fallback) + /** + * Indicates whether the provider is available. + * If not specified, the provider is always available. + * + * May be specified when any of the following criteria is required: + * + * Platform requirements: + * + * - app-* providers are only available on desktop, this is responsible for Tauri runtime checks + * - web-* providers are only available on web, this means Node.js and Tauri should not be imported or used + * + * System spec requirements: + * + * - may requires WebGPU / NVIDIA / other types of GPU, + * on Web, WebGPU will automatically compiled to use targeting GPU hardware + * - may requires significant amount of GPU memory to run, especially for + * using of small language models within browser or Tauri app + * - may requires significant amount of memory to run, especially for those + * non-WebGPU supported environments. + */ + isAvailableBy?: () => Promise | boolean icon?: string iconColor?: string iconImage?: string - defaultOptions?: Record + defaultOptions?: () => Record createProvider: (config: Record) => | ChatProvider | ChatProviderWithExtraOptions @@ -62,6 +86,7 @@ export interface ProviderMetadata { capabilities: { listModels?: (config: Record) => Promise listVoices?: (config: Record) => Promise + loadModel?: (config: Record, hooks?: { onProgress?: (progress: ProgressInfo) => Promise | void }) => Promise } validators: { validateProviderConfig: (config: Record) => Promise | boolean @@ -138,9 +163,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.openrouter.description', description: 'openrouter.ai', icon: 'i-lobe-icons:openrouter', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://openrouter.ai/api/v1/', - }, + }), createProvider: config => createOpenRouter((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listModels: async (config) => { @@ -153,6 +178,212 @@ export const useProvidersStore = defineStore('providers', () => { }, }, }, + 'app-local-audio-speech': { + id: 'app-local-audio-speech', + category: 'speech', + tasks: ['text-to-speech', 'tts'], + isAvailableBy: async () => { + if ('window' in globalThis && globalThis.window != null) { + if ('__TAURI__' in globalThis.window && globalThis.window.__TAURI__ != null) { + return true + } + } + + return false + }, + nameKey: 'settings.pages.providers.provider.app-local-audio-speech.title', + name: 'App (Local)', + descriptionKey: 'settings.pages.providers.provider.app-local-audio-speech.description', + description: 'https://github.com/huggingface/candle', + icon: 'i-lobe-icons:huggingface', + defaultOptions: () => ({}), + createProvider: config => createOpenAI((config.baseUrl as string).trim()), + capabilities: { + listModels: async (config) => { + return (await listModels({ + ...createOpenAI((config.baseUrl as string).trim()).model(), + })).map((model) => { + return { + id: model.id, + name: model.id, + provider: 'app-local-candle', + description: '', + contextLength: 0, + deprecated: false, + } satisfies ModelInfo + }) + }, + }, + validators: { + validateProviderConfig: (config) => { + if (!config.baseUrl) + return false + + // Check if the Ollama server is reachable + return fetch(`${(config.baseUrl as string).trim()}models`) + .then(response => response.ok) + .catch(() => false) + }, + }, + }, + 'app-local-audio-transcription': { + id: 'app-local-audio-transcription', + category: 'transcription', + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + isAvailableBy: async () => { + if ('window' in globalThis && globalThis.window != null) { + if ('__TAURI__' in globalThis.window && globalThis.window.__TAURI__ != null) { + return true + } + } + + return false + }, + nameKey: 'settings.pages.providers.provider.app-local-audio-transcription.title', + name: 'App (Local)', + descriptionKey: 'settings.pages.providers.provider.app-local-audio-transcription.description', + description: 'https://github.com/huggingface/candle', + icon: 'i-lobe-icons:huggingface', + defaultOptions: () => ({}), + createProvider: config => createOpenAI((config.baseUrl as string).trim()), + capabilities: { + listModels: async (config) => { + return (await listModels({ + ...createOpenAI((config.baseUrl as string).trim()).model(), + })).map((model) => { + return { + id: model.id, + name: model.id, + provider: 'app-local-candle', + description: '', + contextLength: 0, + deprecated: false, + } satisfies ModelInfo + }) + }, + }, + validators: { + validateProviderConfig: (config) => { + if (!config.baseUrl) + return false + + // Check if the Ollama server is reachable + return fetch(`${(config.baseUrl as string).trim()}models`) + .then(response => response.ok) + .catch(() => false) + }, + }, + }, + 'browser-local-audio-speech': { + id: 'browser-local-audio-speech', + category: 'speech', + tasks: ['text-to-speech', 'tts'], + isAvailableBy: async () => { + const webGPUAvailable = await isWebGPUSupported() + if (webGPUAvailable) { + return true + } + + if ('navigator' in globalThis && globalThis.navigator != null && 'deviceMemory' in globalThis.navigator && typeof globalThis.navigator.deviceMemory === 'number') { + const memory = globalThis.navigator.deviceMemory + // Check if the device has at least 8GB of RAM + if (memory >= 8) { + return true + } + } + + return false + }, + nameKey: 'settings.pages.providers.provider.browser-local-audio-speech.title', + name: 'Browser (Local)', + descriptionKey: 'settings.pages.providers.provider.browser-local-audio-speech.description', + description: 'https://github.com/moeru-ai/xsai-transformers', + icon: 'i-lobe-icons:huggingface', + defaultOptions: () => ({}), + createProvider: config => createOpenAI((config.baseUrl as string).trim()), + capabilities: { + listModels: async (config) => { + return (await listModels({ + ...createOpenAI((config.baseUrl as string).trim()).model(), + })).map((model) => { + return { + id: model.id, + name: model.id, + provider: 'browser-local-transformers', + description: '', + contextLength: 0, + deprecated: false, + } satisfies ModelInfo + }) + }, + }, + validators: { + validateProviderConfig: (config) => { + if (!config.baseUrl) + return false + + // Check if the Ollama server is reachable + return fetch(`${(config.baseUrl as string).trim()}models`) + .then(response => response.ok) + .catch(() => false) + }, + }, + }, + 'browser-local-audio-transcription': { + id: 'browser-local-audio-transcription', + category: 'transcription', + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + isAvailableBy: async () => { + const webGPUAvailable = await isWebGPUSupported() + if (webGPUAvailable) { + return true + } + + if ('navigator' in globalThis && globalThis.navigator != null && 'deviceMemory' in globalThis.navigator && typeof globalThis.navigator.deviceMemory === 'number') { + const memory = globalThis.navigator.deviceMemory + // Check if the device has at least 8GB of RAM + if (memory >= 8) { + return true + } + } + + return false + }, + nameKey: 'settings.pages.providers.provider.browser-local-audio-transcription.title', + name: 'Browser (Local)', + descriptionKey: 'settings.pages.providers.provider.browser-local-audio-transcription.description', + description: 'https://github.com/moeru-ai/xsai-transformers', + icon: 'i-lobe-icons:huggingface', + defaultOptions: () => ({}), + createProvider: config => createOpenAI((config.baseUrl as string).trim()), + capabilities: { + listModels: async (config) => { + return (await listModels({ + ...createOpenAI((config.baseUrl as string).trim()).model(), + })).map((model) => { + return { + id: model.id, + name: model.id, + provider: 'browser-local-transformers', + description: '', + contextLength: 0, + deprecated: false, + } satisfies ModelInfo + }) + }, + }, + validators: { + validateProviderConfig: (config) => { + if (!config.baseUrl) + return false + + // Check if the Ollama server is reachable + return fetch(`${(config.baseUrl as string).trim()}models`) + .then(response => response.ok) + .catch(() => false) + }, + }, + }, 'ollama': { id: 'ollama', category: 'chat', @@ -162,9 +393,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.ollama.description', description: 'ollama.com', icon: 'i-lobe-icons:ollama', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'http://localhost:11434/v1/', - }, + }), createProvider: config => createOllama((config.baseUrl as string).trim()), capabilities: { listModels: async (config) => { @@ -203,9 +434,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.ollama.description', description: 'ollama.com', icon: 'i-lobe-icons:ollama', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'http://localhost:11434/v1/', - }, + }), createProvider: config => createOllama((config.baseUrl as string).trim()), capabilities: { listModels: async (config) => { @@ -302,9 +533,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.openai.description', description: 'openai.com', icon: 'i-lobe-icons:openai', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://api.openai.com/v1/', - }, + }), createProvider: config => createOpenAI((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listModels: async (config) => { @@ -337,9 +568,44 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.openai.description', description: 'openai.com', icon: 'i-lobe-icons:openai', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://api.openai.com/v1/', + }), + createProvider: config => createOpenAI((config.apiKey as string).trim(), (config.baseUrl as string).trim()), + capabilities: { + listModels: async (config) => { + return (await listModels({ + ...createOpenAI((config.apiKey as string).trim(), (config.baseUrl as string).trim()).model(), + })).map((model) => { + return { + id: model.id, + name: model.id, + provider: 'openai', + description: '', + contextLength: 0, + deprecated: false, + } satisfies ModelInfo + }) + }, }, + validators: { + validateProviderConfig: (config) => { + return !!config.apiKey && !!config.baseUrl + }, + }, + }, + 'openai-audio-transcription': { + id: 'openai-audio-transcription', + category: 'transcription', + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + nameKey: 'settings.pages.providers.provider.openai.title', + name: 'OpenAI', + descriptionKey: 'settings.pages.providers.provider.openai.description', + description: 'openai.com', + icon: 'i-lobe-icons:openai', + defaultOptions: () => ({ + baseUrl: 'https://api.openai.com/v1/', + }), createProvider: config => createOpenAI((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listModels: async (config) => { @@ -372,9 +638,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.anthropic.description', description: 'anthropic.com', icon: 'i-lobe-icons:anthropic', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://api.anthropic.com/v1/', - }, + }), createProvider: config => createAnthropic((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listModels: async () => { @@ -445,9 +711,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.google-generative-ai.description', description: 'ai.google.dev', icon: 'i-lobe-icons:gemini', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://generativelanguage.googleapis.com/v1beta/openai/', - }, + }), createProvider: config => createGoogleGenerativeAI((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listModels: async (config) => { @@ -504,9 +770,9 @@ export const useProvidersStore = defineStore('providers', () => { }, }, 'deepseek': { + id: 'deepseek', category: 'chat', tasks: ['text-generation'], - id: 'deepseek', nameKey: 'settings.pages.providers.provider.deepseek.title', name: 'DeepSeek', descriptionKey: 'settings.pages.providers.provider.deepseek.description', @@ -544,13 +810,13 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.elevenlabs.description', description: 'elevenlabs.io', icon: 'i-simple-icons:elevenlabs', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://unspeech.hyp3r.link/v1/', voiceSettings: { similarityBoost: 0.75, stability: 0.5, }, - }, + }), createProvider: config => createUnElevenLabs((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as SpeechProviderWithExtraOptions, capabilities: { listModels: async () => { @@ -615,9 +881,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.microsoft-speech.description', description: 'speech.microsoft.com', iconColor: 'i-lobe-icons:microsoft', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://unspeech.hyp3r.link/v1/', - }, + }), createProvider: config => createUnMicrosoft((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as SpeechProviderWithExtraOptions, capabilities: { listModels: async () => { @@ -666,9 +932,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.alibaba-cloud-model-studio.description', description: 'bailian.console.aliyun.com', iconColor: 'i-lobe-icons:alibabacloud', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://unspeech.hyp3r.link/v1/', - }, + }), createProvider: config => createUnAlibabaCloud((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listVoices: async (config) => { @@ -725,9 +991,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.volcengine.description', description: 'volcengine.com', iconColor: 'i-lobe-icons:volcengine', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://unspeech.hyp3r.link/v1/', - }, + }), createProvider: config => createUnVolcengine((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listVoices: async (config) => { @@ -872,9 +1138,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.featherless.description', description: 'featherless.ai', icon: 'i-lobe-icons:featherless-ai', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://api.featherless.ai/v1/', - }, + }), createProvider: config => createOpenAI((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listModels: async (config) => { @@ -906,9 +1172,9 @@ export const useProvidersStore = defineStore('providers', () => { name: 'Player2 API', descriptionKey: 'settings.pages.providers.provider.player2.description', description: 'player2.game', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'http://localhost:4315/v1/', - }, + }), createProvider: (config) => { return createPlayer2((config.baseUrl as string).trim()) }, @@ -963,9 +1229,9 @@ export const useProvidersStore = defineStore('providers', () => { descriptionKey: 'settings.pages.providers.provider.perplexity.description', description: 'perplexity.ai', icon: 'i-lobe-icons:perplexity', - defaultOptions: { + defaultOptions: () => ({ baseUrl: 'https://api.perplexity.ai', - }, + }), createProvider: config => createPerplexity((config.apiKey as string).trim(), (config.baseUrl as string).trim()), capabilities: { listModels: async () => { @@ -1100,8 +1366,9 @@ export const useProvidersStore = defineStore('providers', () => { function initializeProvider(providerId: string) { if (!providerCredentials.value[providerId]) { const metadata = providerMetadata[providerId] + const defaultOptions = metadata.defaultOptions?.() || {} providerCredentials.value[providerId] = { - baseUrl: metadata.defaultOptions?.baseUrl || '', + baseUrl: defaultOptions.baseUrl || '', } } } @@ -1236,20 +1503,32 @@ export const useProvidersStore = defineStore('providers', () => { } } - const availableProvidersMetadata = computed(() => { - return availableProviders.value.map(id => getProviderMetadata(id)) - }) + const availableProvidersMetadata = computedAsync(async () => { + const providers: ProviderMetadata[] = [] + + for (const provider of allProvidersMetadata.value) { + const p = getProviderMetadata(provider.id) + const isAvailableBy = p.isAvailableBy || (() => true) + + const isAvailable = await isAvailableBy() + if (isAvailable) { + providers.push(provider) + } + } + + return providers + }, []) const allChatProvidersMetadata = computed(() => { - return allProvidersMetadata.value.filter(metadata => metadata.category === 'chat') + return availableProvidersMetadata.value.filter(metadata => metadata.category === 'chat') }) const allAudioSpeechProvidersMetadata = computed(() => { - return allProvidersMetadata.value.filter(metadata => metadata.category === 'speech') + return availableProvidersMetadata.value.filter(metadata => metadata.category === 'speech') }) const allAudioTranscriptionProvidersMetadata = computed(() => { - return allProvidersMetadata.value.filter(metadata => metadata.category === 'transcription') + return availableProvidersMetadata.value.filter(metadata => metadata.category === 'transcription') }) const configuredChatProvidersMetadata = computed(() => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e1303cc57..88f9917c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1009,6 +1009,9 @@ importers: '@xsai-transformers/embed': specifier: ^0.0.6 version: 0.0.6(web-worker@1.5.0) + '@xsai-transformers/shared': + specifier: ^0.0.6 + version: 0.0.6 '@xsai/embed': specifier: 'catalog:' version: 0.3.0-beta.4 @@ -1033,6 +1036,9 @@ importers: defu: specifier: ^6.1.4 version: 6.1.4 + gpuu: + specifier: ^1.0.3 + version: 1.0.3 jszip: specifier: ^3.10.1 version: 3.10.1