Revert "fix(stage-ui): correct stream reuse when switching sources + improve capture pipeline" (#1741)

Reverts moeru-ai/airi#1731
This commit is contained in:
Neko
2026-04-27 15:22:19 +08:00
committed by GitHub
parent 4ba5b13c0d
commit 0c8c99dae3
7 changed files with 127 additions and 191 deletions
@@ -93,16 +93,4 @@ describe('vision orchestrator', () => {
contextId: 'vision:screen:understand',
})
})
it('records inference failures on the store before rethrowing', async () => {
const store = useVisionOrchestratorStore()
runVisionInference.mockRejectedValueOnce(new Error('Vision inference failed'))
await expect(store.processCapture({
imageDataUrl: 'data:image/jpeg;base64,broken',
workloadId: 'screen:interpret',
})).rejects.toThrow('Vision inference failed')
expect(store.lastError).toBe('Vision inference failed')
})
})
@@ -2,7 +2,6 @@ import type { CommonContentPart } from '@xsai/shared-chat'
import type { VisionWorkloadId } from '../../../composables/vision/use-vision-workloads'
import { errorMessageFrom } from '@moeru/std'
import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
import { defineStore, storeToRefs } from 'pinia'
import { ref } from 'vue'
@@ -12,19 +11,11 @@ import { getVisionWorkload } from '../../../composables/vision/use-vision-worklo
import { useModsServerChannelStore } from '../../mods/api/channel-server'
import { useVisionStore } from './store'
/**
* Payload describing one captured frame routed through the vision orchestrator.
*/
export interface VisionCapturePayload {
/** JPEG or PNG data URL captured from the selected source. */
imageDataUrl: string
/** Vision workload that describes how the frame should be interpreted. */
workloadId: VisionWorkloadId
/** Optional source identifier used to keep context updates stable per source. */
sourceId?: string
/** Timestamp recorded when the frame was captured. */
capturedAt?: number
/** When `true`, publish the inference result into the character context channel. */
publishContext?: boolean
}
@@ -34,19 +25,6 @@ function getVisionContextId(payload: Pick<VisionCapturePayload, 'workloadId' | '
: `vision:${payload.workloadId}`
}
/**
* Coordinates screen-capture inference and optional context publishing for vision workflows.
*
* Use when:
* - A renderer page captures frames and needs multimodal inference results
* - Successful results may also need to become context updates for downstream modules
*
* Expects:
* - The vision settings store to already contain an active provider and model
*
* Returns:
* - A Pinia store that tracks the latest result, last error, and capture-processing actions
*/
export const useVisionOrchestratorStore = defineStore('vision-orchestrator', () => {
const visionStore = useVisionStore()
const { activeProvider, activeModel } = storeToRefs(visionStore)
@@ -59,64 +37,55 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', ()
const lastWorkloadId = ref<VisionWorkloadId>('screen:interpret')
async function processCapture(payload: VisionCapturePayload) {
if (!activeProvider.value || !activeModel.value) {
const configurationError = new Error('Vision model is not configured')
recordError(configurationError)
throw configurationError
}
if (!activeProvider.value || !activeModel.value)
throw new Error('Vision model is not configured')
lastWorkloadId.value = payload.workloadId
try {
const text = await runVisionInference({
imageDataUrl: payload.imageDataUrl,
workloadId: payload.workloadId,
const text = await runVisionInference({
imageDataUrl: payload.imageDataUrl,
workloadId: payload.workloadId,
})
lastResultText.value = text
lastResultAt.value = Date.now()
lastError.value = null
if (payload.publishContext) {
const workload = getVisionWorkload(payload.workloadId)
const content: CommonContentPart[] = [
{ type: 'text', text },
{
type: 'image_url',
image_url: {
url: payload.imageDataUrl,
},
},
]
modsServerChannelStore.sendContextUpdate({
strategy: ContextUpdateStrategy.ReplaceSelf,
contextId: getVisionContextId(payload),
text,
content,
metadata: {
module: 'vision',
workload: workload.id,
workloadLabel: workload.label,
sourceId: payload.sourceId,
capturedAt: payload.capturedAt,
provider: activeProvider.value,
model: activeModel.value,
},
})
lastResultText.value = text
lastResultAt.value = Date.now()
lastError.value = null
if (payload.publishContext) {
const workload = getVisionWorkload(payload.workloadId)
const content: CommonContentPart[] = [
{ type: 'text', text },
{
type: 'image_url',
image_url: {
url: payload.imageDataUrl,
},
},
]
modsServerChannelStore.sendContextUpdate({
strategy: ContextUpdateStrategy.ReplaceSelf,
contextId: getVisionContextId(payload),
text,
content,
metadata: {
module: 'vision',
workload: workload.id,
workloadLabel: workload.label,
sourceId: payload.sourceId,
capturedAt: payload.capturedAt,
provider: activeProvider.value,
model: activeModel.value,
},
})
return { contextUpdates: 1, text }
}
return { contextUpdates: 0, text }
}
catch (error) {
recordError(error)
throw error
return { contextUpdates: 1, text }
}
return { contextUpdates: 0, text }
}
function recordError(error: unknown) {
lastError.value = errorMessageFrom(error) ?? 'Unknown error'
lastError.value = error instanceof Error ? error.message : String(error)
}
return {