diff --git a/apps/stage-web/src/pages/settings/modules/hearing.vue b/apps/stage-web/src/pages/settings/modules/hearing.vue index 17b0c1feb..346d32621 100644 --- a/apps/stage-web/src/pages/settings/modules/hearing.vue +++ b/apps/stage-web/src/pages/settings/modules/hearing.vue @@ -4,9 +4,9 @@ import { FieldCheckbox, FieldRange, FieldSelect } from '@proj-airi/ui' import { useDevicesList } from '@vueuse/core' import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue' -import workletUrl from '../../../utils/vad/process.worklet?worker&url' +import workletUrl from '../../../workers/vad/process.worklet?worker&url' -import { createVAD, VADAudioManager } from '../../../utils/vad' +import { createVAD, VADAudioManager } from '../../../workers/vad' const devices = useDevicesList({ constraints: { audio: true } }) const audioInputs = computed(() => devices.audioInputs.value) diff --git a/apps/stage-web/src/workers/jsonFormat.ts b/apps/stage-web/src/workers/jsonFormat.ts new file mode 100644 index 000000000..36bb17078 --- /dev/null +++ b/apps/stage-web/src/workers/jsonFormat.ts @@ -0,0 +1,74 @@ +import type { CommonRequestOptions } from '@xsai/shared' +import type { Message } from '@xsai/shared-chat' +import type { Infer, Schema } from 'xsschema' + +import { generateText } from '@xsai/generate-text' +import { message } from '@xsai/utils-chat' +import { toJsonSchema, validate } from 'xsschema' + +type SchemaOrString = S extends unknown ? string : S extends Schema ? Infer : never + +async function parseJSONFormat>(content: string, options: { messages: Message[], apiKey?: string, baseURL: string, model: string } & Partial, schema?: S, erroredValue?: string, errorMessage?: string): Promise { + if (!schema) + return content as unknown as R + + try { + let parsedContent: Infer + let correctionPrompt = '' + + if (erroredValue && errorMessage) { + correctionPrompt = `Previous response "${JSON.stringify(erroredValue)}" was invalid due to: ${JSON.stringify(errorMessage)}\n\n` + } + + try { + parsedContent = JSON.parse(content) + } + catch (parseError) { + console.error('Error parsing JSON:', parseError, content) + + options.messages.push(message.user(` +${correctionPrompt}The response was not valid JSON: +${JSON.stringify(content)} + +Error: ${String(parseError)} + +Please provide a corrected JSON response that matches the schema: +${JSON.stringify(await toJsonSchema(schema))}`)) + + const response = await call(options, schema) + return parseJSONFormat(response, options, schema, content, String(parseError)) + } + + // TODO: print validation issues + return await validate(schema, parsedContent) as R + } + catch (error) { + console.error('Error processing response:', error) + throw error + } +} + +/** + * Processes user input and generates LLM response along with thought nodes. + */ +async function call>(options: { messages: Message[], apiKey?: string, baseURL: string, model: string } & Partial, schema?: S): Promise { + if (schema != null) { + options.messages.push(message.user(`Your response must follow the following schema: +${JSON.stringify(await toJsonSchema(schema))} + +Without any extra markups such as \`\`\` in markdown, or descriptions.`)) + } + + const response = await generateText({ + baseURL: options.baseURL, + apiKey: options.apiKey, + model: options.model, + messages: options.messages, + }) + + return await parseJSONFormat(response.text || '', options, schema) +} + +export async function generateObject>(options: { messages: Message[], model: string, apiKey?: string, baseURL: string } & Partial, schema?: S): Promise { + return await call(options, schema) +} diff --git a/apps/stage-web/src/workers/platform.ts b/apps/stage-web/src/workers/platform.ts new file mode 100644 index 000000000..7eed72ae0 --- /dev/null +++ b/apps/stage-web/src/workers/platform.ts @@ -0,0 +1,3 @@ +export function isPlatformTamagotchi() { + return import.meta.env.MODE === 'tamagotchi' +} diff --git a/apps/stage-web/src/utils/vad/index.ts b/apps/stage-web/src/workers/vad/index.ts similarity index 100% rename from apps/stage-web/src/utils/vad/index.ts rename to apps/stage-web/src/workers/vad/index.ts diff --git a/apps/stage-web/src/utils/vad/manager.ts b/apps/stage-web/src/workers/vad/manager.ts similarity index 100% rename from apps/stage-web/src/utils/vad/manager.ts rename to apps/stage-web/src/workers/vad/manager.ts diff --git a/apps/stage-web/src/utils/vad/process.worklet.ts b/apps/stage-web/src/workers/vad/process.worklet.ts similarity index 100% rename from apps/stage-web/src/utils/vad/process.worklet.ts rename to apps/stage-web/src/workers/vad/process.worklet.ts diff --git a/apps/stage-web/src/utils/vad/vad.ts b/apps/stage-web/src/workers/vad/vad.ts similarity index 98% rename from apps/stage-web/src/utils/vad/vad.ts rename to apps/stage-web/src/workers/vad/vad.ts index 9b696044f..52c3059f3 100644 --- a/apps/stage-web/src/utils/vad/vad.ts +++ b/apps/stage-web/src/workers/vad/vad.ts @@ -1,3 +1,5 @@ +import type { PreTrainedModel } from '@huggingface/transformers' + import { AutoModel, Tensor } from '@huggingface/transformers' // Default configuration parameters @@ -41,7 +43,7 @@ export type VADEventCallback */ export class VAD { private config: VADConfig - private model: any + private model: PreTrainedModel | undefined private state: Tensor private sampleRateTensor: Tensor private buffer: Float32Array @@ -68,14 +70,9 @@ export class VAD { this.config = { ...defaultConfig, ...userConfig } - // Create buffer based on max duration this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate) - - // Initialize state tensor for VAD model - this.state = new Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128]) - - // Sample rate tensor for the model this.sampleRateTensor = new Tensor('int64', [this.config.sampleRate], []) + this.state = new Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128]) } /** @@ -215,7 +212,7 @@ export class VAD { const input = new Tensor('float32', buffer, [1, buffer.length]) const { stateN, output } = await (this.inferenceChain = this.inferenceChain.then(() => - this.model({ + this.model?.({ input, sr: this.sampleRateTensor, state: this.state,