Revert "fix(stage-ui): correct stream reuse when switching sources + improve capture pipeline" (#1741)
Reverts moeru-ai/airi#1731
This commit is contained in:
@@ -93,16 +93,4 @@ describe('vision orchestrator', () => {
|
||||
contextId: 'vision:screen:understand',
|
||||
})
|
||||
})
|
||||
|
||||
it('records inference failures on the store before rethrowing', async () => {
|
||||
const store = useVisionOrchestratorStore()
|
||||
runVisionInference.mockRejectedValueOnce(new Error('Vision inference failed'))
|
||||
|
||||
await expect(store.processCapture({
|
||||
imageDataUrl: 'data:image/jpeg;base64,broken',
|
||||
workloadId: 'screen:interpret',
|
||||
})).rejects.toThrow('Vision inference failed')
|
||||
|
||||
expect(store.lastError).toBe('Vision inference failed')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,7 +2,6 @@ import type { CommonContentPart } from '@xsai/shared-chat'
|
||||
|
||||
import type { VisionWorkloadId } from '../../../composables/vision/use-vision-workloads'
|
||||
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
|
||||
import { defineStore, storeToRefs } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
@@ -12,19 +11,11 @@ import { getVisionWorkload } from '../../../composables/vision/use-vision-worklo
|
||||
import { useModsServerChannelStore } from '../../mods/api/channel-server'
|
||||
import { useVisionStore } from './store'
|
||||
|
||||
/**
|
||||
* Payload describing one captured frame routed through the vision orchestrator.
|
||||
*/
|
||||
export interface VisionCapturePayload {
|
||||
/** JPEG or PNG data URL captured from the selected source. */
|
||||
imageDataUrl: string
|
||||
/** Vision workload that describes how the frame should be interpreted. */
|
||||
workloadId: VisionWorkloadId
|
||||
/** Optional source identifier used to keep context updates stable per source. */
|
||||
sourceId?: string
|
||||
/** Timestamp recorded when the frame was captured. */
|
||||
capturedAt?: number
|
||||
/** When `true`, publish the inference result into the character context channel. */
|
||||
publishContext?: boolean
|
||||
}
|
||||
|
||||
@@ -34,19 +25,6 @@ function getVisionContextId(payload: Pick<VisionCapturePayload, 'workloadId' | '
|
||||
: `vision:${payload.workloadId}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Coordinates screen-capture inference and optional context publishing for vision workflows.
|
||||
*
|
||||
* Use when:
|
||||
* - A renderer page captures frames and needs multimodal inference results
|
||||
* - Successful results may also need to become context updates for downstream modules
|
||||
*
|
||||
* Expects:
|
||||
* - The vision settings store to already contain an active provider and model
|
||||
*
|
||||
* Returns:
|
||||
* - A Pinia store that tracks the latest result, last error, and capture-processing actions
|
||||
*/
|
||||
export const useVisionOrchestratorStore = defineStore('vision-orchestrator', () => {
|
||||
const visionStore = useVisionStore()
|
||||
const { activeProvider, activeModel } = storeToRefs(visionStore)
|
||||
@@ -59,64 +37,55 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', ()
|
||||
const lastWorkloadId = ref<VisionWorkloadId>('screen:interpret')
|
||||
|
||||
async function processCapture(payload: VisionCapturePayload) {
|
||||
if (!activeProvider.value || !activeModel.value) {
|
||||
const configurationError = new Error('Vision model is not configured')
|
||||
recordError(configurationError)
|
||||
throw configurationError
|
||||
}
|
||||
if (!activeProvider.value || !activeModel.value)
|
||||
throw new Error('Vision model is not configured')
|
||||
|
||||
lastWorkloadId.value = payload.workloadId
|
||||
|
||||
try {
|
||||
const text = await runVisionInference({
|
||||
imageDataUrl: payload.imageDataUrl,
|
||||
workloadId: payload.workloadId,
|
||||
const text = await runVisionInference({
|
||||
imageDataUrl: payload.imageDataUrl,
|
||||
workloadId: payload.workloadId,
|
||||
})
|
||||
|
||||
lastResultText.value = text
|
||||
lastResultAt.value = Date.now()
|
||||
lastError.value = null
|
||||
|
||||
if (payload.publishContext) {
|
||||
const workload = getVisionWorkload(payload.workloadId)
|
||||
const content: CommonContentPart[] = [
|
||||
{ type: 'text', text },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: payload.imageDataUrl,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
modsServerChannelStore.sendContextUpdate({
|
||||
strategy: ContextUpdateStrategy.ReplaceSelf,
|
||||
contextId: getVisionContextId(payload),
|
||||
text,
|
||||
content,
|
||||
metadata: {
|
||||
module: 'vision',
|
||||
workload: workload.id,
|
||||
workloadLabel: workload.label,
|
||||
sourceId: payload.sourceId,
|
||||
capturedAt: payload.capturedAt,
|
||||
provider: activeProvider.value,
|
||||
model: activeModel.value,
|
||||
},
|
||||
})
|
||||
|
||||
lastResultText.value = text
|
||||
lastResultAt.value = Date.now()
|
||||
lastError.value = null
|
||||
|
||||
if (payload.publishContext) {
|
||||
const workload = getVisionWorkload(payload.workloadId)
|
||||
const content: CommonContentPart[] = [
|
||||
{ type: 'text', text },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: payload.imageDataUrl,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
modsServerChannelStore.sendContextUpdate({
|
||||
strategy: ContextUpdateStrategy.ReplaceSelf,
|
||||
contextId: getVisionContextId(payload),
|
||||
text,
|
||||
content,
|
||||
metadata: {
|
||||
module: 'vision',
|
||||
workload: workload.id,
|
||||
workloadLabel: workload.label,
|
||||
sourceId: payload.sourceId,
|
||||
capturedAt: payload.capturedAt,
|
||||
provider: activeProvider.value,
|
||||
model: activeModel.value,
|
||||
},
|
||||
})
|
||||
return { contextUpdates: 1, text }
|
||||
}
|
||||
|
||||
return { contextUpdates: 0, text }
|
||||
}
|
||||
catch (error) {
|
||||
recordError(error)
|
||||
throw error
|
||||
return { contextUpdates: 1, text }
|
||||
}
|
||||
|
||||
return { contextUpdates: 0, text }
|
||||
}
|
||||
|
||||
function recordError(error: unknown) {
|
||||
lastError.value = errorMessageFrom(error) ?? 'Unknown error'
|
||||
lastError.value = error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user