From 2c4ac66b61fdc97fbac06414e7752c27e23c9780 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Mon, 23 Mar 2026 13:45:47 +0800 Subject: [PATCH] chore(stage-ui): lint and timeout constraint --- .../vision/use-vision-inference.test.ts | 93 +++++++++++++++++++ .../vision/use-vision-inference.ts | 36 +++++-- .../stores/modules/vision/processing-store.ts | 3 +- pnpm-workspace.yaml | 36 +++---- 4 files changed, 142 insertions(+), 26 deletions(-) create mode 100644 packages/stage-ui/src/composables/vision/use-vision-inference.test.ts diff --git a/packages/stage-ui/src/composables/vision/use-vision-inference.test.ts b/packages/stage-ui/src/composables/vision/use-vision-inference.test.ts new file mode 100644 index 000000000..63b99d865 --- /dev/null +++ b/packages/stage-ui/src/composables/vision/use-vision-inference.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { reactive, toRefs } from 'vue' + +const stream = vi.fn() +const getProviderInstance = vi.fn() + +vi.mock('pinia', async () => { + const actual = await vi.importActual('pinia') + return { + ...actual, + storeToRefs: (store: object) => toRefs(store as never), + } +}) + +vi.mock('../../stores/llm', () => ({ + useLLM: () => ({ + stream, + }), +})) + +vi.mock('../../stores/providers', () => ({ + useProvidersStore: () => ({ + getProviderInstance, + }), +})) + +vi.mock('../../stores/modules/vision', () => ({ + useVisionStore: () => reactive({ + activeProvider: 'mock-provider', + activeModel: 'mock-model', + ollamaThinkingEnabled: false, + }), +})) + +vi.mock('./use-vision-workloads', () => ({ + getVisionWorkload: () => ({ + prompt: 'Interpret this frame', + }), +})) + +describe('useVisionInference', () => { + beforeEach(() => { + vi.useFakeTimers() + stream.mockReset() + getProviderInstance.mockReset() + getProviderInstance.mockResolvedValue({ + chat: vi.fn().mockReturnValue({ + apiKey: 'test-key', + baseURL: 'https://example.com/v1/', + }), + }) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('passes an abort signal to llmStore.stream', async () => { + stream.mockImplementation(async (_model, _provider, _messages, options) => { + expect(options?.abortSignal).toBeInstanceOf(AbortSignal) + options?.onStreamEvent?.({ type: 'text-delta', text: 'Frame summary' }) + }) + + const { useVisionInference } = await import('./use-vision-inference') + const { runVisionInference } = useVisionInference() + + await expect(runVisionInference({ + imageDataUrl: 'data:image/png;base64,Zm9v', + workloadId: 'screen:interpret', + })).resolves.toBe('Frame summary') + }) + + it('aborts vision inference when the stream never settles', async () => { + stream.mockImplementation((_model, _provider, _messages, options) => new Promise((_, reject) => { + options?.abortSignal?.addEventListener('abort', () => { + reject(options.abortSignal?.reason) + }, { once: true }) + })) + + const { useVisionInference } = await import('./use-vision-inference') + const { runVisionInference } = useVisionInference() + + const result = runVisionInference({ + imageDataUrl: 'data:image/png;base64,Zm9v', + workloadId: 'screen:interpret', + }) + const expectation = expect(result).rejects.toThrow('Vision inference timed out after 15000ms') + + await vi.advanceTimersByTimeAsync(15_000) + + await expectation + }) +}) diff --git a/packages/stage-ui/src/composables/vision/use-vision-inference.ts b/packages/stage-ui/src/composables/vision/use-vision-inference.ts index 417b846fa..849706db4 100644 --- a/packages/stage-ui/src/composables/vision/use-vision-inference.ts +++ b/packages/stage-ui/src/composables/vision/use-vision-inference.ts @@ -17,6 +17,9 @@ export interface VisionInferenceInput { promptOverride?: string } +// TODO: this should be configurable +const VISION_INFERENCE_TIMEOUT_MS = 60_000 + function parseDataUrl(dataUrl: string) { if (!dataUrl.startsWith('data:')) return { mimeType: 'image/png', base64: dataUrl, url: dataUrl } @@ -74,13 +77,32 @@ export function useVisionInference() { ] let buffer = '' - await llmStore.stream(activeModel.value, visionProvider, messages, { - onStreamEvent: (event) => { - if (event.type === 'text-delta') { - buffer += event.text - } - }, - }) + const abortController = new AbortController() + const timeoutHandle = setTimeout(() => { + abortController.abort(new Error(`Vision inference timed out after ${VISION_INFERENCE_TIMEOUT_MS}ms`)) + }, VISION_INFERENCE_TIMEOUT_MS) + + try { + await llmStore.stream(activeModel.value, visionProvider, messages, { + abortSignal: abortController.signal, + onStreamEvent: (event) => { + if (event.type === 'text-delta') { + buffer += event.text + } + }, + }) + } + catch (error) { + if (abortController.signal.aborted) { + throw abortController.signal.reason instanceof Error + ? abortController.signal.reason + : new Error(`Vision inference timed out after ${VISION_INFERENCE_TIMEOUT_MS}ms`) + } + throw error + } + finally { + clearTimeout(timeoutHandle) + } lastText.value = buffer.trim() return lastText.value diff --git a/packages/stage-ui/src/stores/modules/vision/processing-store.ts b/packages/stage-ui/src/stores/modules/vision/processing-store.ts index 7b4afa40b..b53980081 100644 --- a/packages/stage-ui/src/stores/modules/vision/processing-store.ts +++ b/packages/stage-ui/src/stores/modules/vision/processing-store.ts @@ -1,3 +1,4 @@ +import { errorMessageFrom } from '@moeru/std' import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables' import { defineStore } from 'pinia' import { computed, ref, watch } from 'vue' @@ -112,7 +113,7 @@ export const useVisionProcessingStore = defineStore('vision-processing', () => { recordContextUpdates(outcome.contextUpdates) } catch (error) { - lastError.value = error instanceof Error ? error.message : String(error) + lastError.value = errorMessageFrom(error) || 'Unknown error' } finally { recordProcessingDuration(performance.now() - start) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index bc3c219bc..70923fdae 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,5 @@ +catalogMode: prefer +shellEmulator: true packages: - packages/** - plugins/** @@ -7,6 +9,22 @@ packages: - docs/** - apps/** - '!**/dist/**' +overrides: + array-flatten: npm:@nolyfill/array-flatten@^1.0.44 + axios: npm:feaxios@^0.0.23 + is-core-module: npm:@nolyfill/is-core-module@^1.0.39 + isarray: npm:@nolyfill/isarray@^1.0.44 + onnxruntime-web: npm:onnxruntime-web@^1.24.3 + safe-buffer: npm:@nolyfill/safe-buffer@^1.0.44 + safer-buffer: npm:@nolyfill/safer-buffer@^1.0.44 + side-channel: npm:@nolyfill/side-channel@^1.0.44 + string.prototype.matchall: npm:@nolyfill/string.prototype.matchall@^1.0.44 +patchedDependencies: + '@mediapipe/tasks-vision': patches/@mediapipe__tasks-vision.patch + mineflayer-pathfinder: patches/mineflayer-pathfinder.patch + mineflayer@4.33.0: patches/mineflayer@4.33.0.patch + pixi-live2d-display: patches/pixi-live2d-display.patch + srvx: patches/srvx.patch catalog: '@capacitor/android': ^8.2.0 '@capacitor/cli': ^8.2.0 @@ -89,7 +107,6 @@ catalog: xsschema: 0.4.0-beta.13 yaml: ^2.8.3 zod: ^4.3.6 -catalogMode: prefer catalogs: vitest: '@vitest/browser-playwright': ^4.1.0 @@ -127,16 +144,6 @@ onlyBuiltDependencies: - spawn-sync - utf-8-validate - vue-demi -overrides: - array-flatten: npm:@nolyfill/array-flatten@^1.0.44 - axios: npm:feaxios@^0.0.23 - is-core-module: npm:@nolyfill/is-core-module@^1.0.39 - isarray: npm:@nolyfill/isarray@^1.0.44 - onnxruntime-web: npm:onnxruntime-web@^1.24.3 - safe-buffer: npm:@nolyfill/safe-buffer@^1.0.44 - safer-buffer: npm:@nolyfill/safer-buffer@^1.0.44 - side-channel: npm:@nolyfill/side-channel@^1.0.44 - string.prototype.matchall: npm:@nolyfill/string.prototype.matchall@^1.0.44 packageExtensions: '@formkit/auto-animate': peerDependencies: @@ -168,10 +175,3 @@ packageExtensions: 'vue-sonner': peerDependencies: vue: '^3.2.0' -patchedDependencies: - '@mediapipe/tasks-vision': patches/@mediapipe__tasks-vision.patch - mineflayer-pathfinder: patches/mineflayer-pathfinder.patch - mineflayer@4.33.0: patches/mineflayer@4.33.0.patch - pixi-live2d-display: patches/pixi-live2d-display.patch - srvx: patches/srvx.patch -shellEmulator: true