From 0c8c99dae3fbdfeaf83d8cbcef2df42c79347d70 Mon Sep 17 00:00:00 2001 From: Neko Date: Mon, 27 Apr 2026 15:22:19 +0800 Subject: [PATCH] Revert "fix(stage-ui): correct stream reuse when switching sources + improve capture pipeline" (#1741) Reverts moeru-ai/airi#1731 --- .../composables/use-vision-screen-capture.ts | 118 ++++++++---------- .../pages/devtools/screen-capture.vue | 19 ++- .../src/renderer/pages/devtools/vision.vue | 37 +++--- .../utils/create-object-url-from-bytes.ts | 17 --- nix/assets-hash.txt | 2 +- .../modules/vision/orchestrator.test.ts | 12 -- .../src/stores/modules/vision/orchestrator.ts | 113 ++++++----------- 7 files changed, 127 insertions(+), 191 deletions(-) delete mode 100644 apps/stage-tamagotchi/src/renderer/utils/create-object-url-from-bytes.ts diff --git a/apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts b/apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts index 571fa272c..3df35bdec 100644 --- a/apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts +++ b/apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts @@ -3,46 +3,35 @@ import type { SourcesOptions } from 'electron' import type { MaybeRefOrGetter } from 'vue' import { useElectronScreenCapture } from '@proj-airi/electron-screen-capture/vue' -import { computed, ref, shallowRef, watch } from 'vue' - -import { createObjectUrlFromBytes } from '../utils/create-object-url-from-bytes' +import { computed, ref, toRaw, toValue } from 'vue' interface ScreenCaptureSource extends SerializableDesktopCapturerSource { appIconURL?: string thumbnailURL?: string } -/** - * Manages Electron-backed screen-capture sources and the active preview stream for vision workflows. - * - * Use when: - * - A renderer page needs to browse screen/window sources before capturing frames - * - The page should keep a single active `MediaStream` in sync with the selected source - * - * Expects: - * - The Electron screen-capture preload APIs to be available on `window.electron.ipcRenderer` - * - Callers to invoke `cleanup()` when the owning component unmounts - * - * Returns: - * - Reactive source lists, active stream state, and helpers for refetching, starting, stopping, and capturing frames - */ +function toLocalArrayBuffer(bytes: Uint8Array) { + if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) { + return bytes.slice().buffer + } + return bytes.buffer as ArrayBuffer +} + +function toObjectUrl(bytes: Uint8Array, mime: string) { + return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime })) +} + export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter) { const sources = ref([]) const isRefetching = ref(false) const hasFetchedOnce = ref(false) const activeSourceId = ref('') - const activeStream = shallowRef(null) - const activeStreamSourceId = ref('') - - watch(activeSourceId, (nextId) => { - if (activeStreamSourceId.value && activeStreamSourceId.value !== nextId) { - clearActiveStream() - } - }) + const activeStream = ref(null) const { getSources, - selectWithSource, + setSource, + resetSource, } = useElectronScreenCapture(window.electron.ipcRenderer, sourcesOptions) const activeSource = computed(() => sources.value.find(source => source.id === activeSourceId.value) || null) @@ -58,31 +47,18 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter track.stop()) activeStream.value = null - activeStreamSourceId.value = '' } - function revokeSourceObjectUrls(entries: ScreenCaptureSource[]) { - entries.forEach((source) => { - if (source.appIconURL) - URL.revokeObjectURL(source.appIconURL) - if (source.thumbnailURL) - URL.revokeObjectURL(source.thumbnailURL) - }) - } - - function attachStreamLifecycle(stream: MediaStream, sourceId: string) { + function attachStreamLifecycle(stream: MediaStream) { stream.getTracks().forEach((track) => { track.addEventListener('ended', () => { - if (activeStream.value === stream && activeStreamSourceId.value === sourceId) { + if (activeStream.value === stream) activeStream.value = null - activeStreamSourceId.value = '' - } }, { once: true }) }) } @@ -99,17 +75,22 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter { + if (oldSource.appIconURL) + URL.revokeObjectURL(oldSource.appIconURL) + if (oldSource.thumbnailURL) + URL.revokeObjectURL(oldSource.thumbnailURL) + }) sources.value = nextSources.map(source => ({ ...source, - appIconURL: source.appIcon && source.appIcon.length > 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined, - thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined, + appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined, + thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined, })) const hasActiveSource = sources.value.some(source => source.id === activeSourceId.value) - const nextActiveSourceId = hasActiveSource ? activeSourceId.value : sources.value[0]?.id || '' - activeSourceId.value = nextActiveSourceId + if (!hasActiveSource) + activeSourceId.value = sources.value[0]?.id || '' } finally { isRefetching.value = false @@ -118,29 +99,32 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter sourceId, - async () => await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }), - ) - if (!isActiveStream(stream)) { - stream.getTracks().forEach(track => track.stop()) - throw new Error('Selected source did not provide a live video track') + const handle = await setSource({ + options: toRaw(toValue(sourcesOptions)), + sourceId: activeSourceId.value, + }) + + try { + const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }) + activeStream.value = stream + attachStreamLifecycle(stream) + return stream + } + catch (error) { + activeStream.value = null + throw error + } + finally { + await resetSource(handle) } - - activeStream.value = stream - activeStreamSourceId.value = sourceId - attachStreamLifecycle(stream, sourceId) - - return stream } function stopStream() { @@ -149,7 +133,12 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter { + if (oldSource.appIconURL) + URL.revokeObjectURL(oldSource.appIconURL) + if (oldSource.thumbnailURL) + URL.revokeObjectURL(oldSource.thumbnailURL) + }) } function captureFrame(video: HTMLVideoElement, quality = 0.82, maxWidth = 1280, maxHeight = 720) { @@ -159,9 +148,6 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined, - thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined, + appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined, + thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined, })) } catch (err) { @@ -144,7 +153,7 @@ async function refetchSources() { } onMounted(async () => { - await refetchSources() + refetchSources() }) onBeforeUnmount(() => { diff --git a/apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue b/apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue index 0645ce292..e66137c6f 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue @@ -8,13 +8,13 @@ import { VISION_WORKLOADS } from '@proj-airi/stage-ui/composables' import { useVisionOrchestratorStore, useVisionProcessingStore, useVisionStore } from '@proj-airi/stage-ui/stores/modules/vision' import { Button, FieldCheckbox, FieldCombobox, FieldRange, SelectTab } from '@proj-airi/ui' import { storeToRefs } from 'pinia' -import { computed, onBeforeUnmount, onMounted, ref } from 'vue' +import { computed, onBeforeUnmount, ref } from 'vue' import WithScreenCapture from '../../components/WithScreenCapture.vue' import { useVisionScreenCapture } from '../../composables/use-vision-screen-capture' -type SourceCategory = 'applications' | 'displays' +type SourceCategory = 'applications' | 'displays' | 'devices' const visionStore = useVisionStore() const visionProcessingStore = useVisionProcessingStore() @@ -68,6 +68,7 @@ const { const categoryOptions = [ { label: 'Applications', value: 'applications', icon: 'i-solar:window-frame-line-duotone' }, { label: 'Displays', value: 'displays', icon: 'i-solar:screencast-2-line-duotone' }, + { label: 'Devices', value: 'devices', icon: 'i-solar:smartphone-2-line-duotone' }, ] const workloadOptions = VISION_WORKLOADS.map(workload => ({ @@ -77,21 +78,27 @@ const workloadOptions = VISION_WORKLOADS.map(workload => ({ const isDisplaySource = (source: { id: string }) => source.id.startsWith('screen:') const isWindowSource = (source: { id: string }) => source.id.startsWith('window:') +const isDeviceSource = (source: { id: string }) => source.id.startsWith('device:') const filteredSources = computed(() => { if (sourceCategory.value === 'applications') return sources.value.filter(isWindowSource) - return sources.value.filter(isDisplaySource) + if (sourceCategory.value === 'displays') + return sources.value.filter(isDisplaySource) + return sources.value.filter(isDeviceSource) }) const sourceCounts = computed(() => ({ applications: sources.value.filter(isWindowSource).length, displays: sources.value.filter(isDisplaySource).length, + devices: sources.value.filter(isDeviceSource).length, })) function getShareLabel(source: { id: string }) { if (isDisplaySource(source)) return 'Share Screen' + if (isDeviceSource(source)) + return 'Share Device' return 'Share Window' } @@ -147,13 +154,7 @@ async function ensureVideoStream() { resolve() return } - - const handleLoadedMetadata = () => { - video.removeEventListener('loadedmetadata', handleLoadedMetadata) - resolve() - } - - video.addEventListener('loadedmetadata', handleLoadedMetadata) + video.onloadedmetadata = () => resolve() }) } @@ -163,7 +164,7 @@ async function handleVisionTick() { try { if (!hasLiveVideoStream(activeStream.value)) { - stopStream() + activeStream.value = null await ensureVideoStream() } @@ -252,10 +253,6 @@ async function shareSource(sourceId: string) { } } -onMounted(() => { - void refetchSources() -}) - onBeforeUnmount(() => { visionProcessingStore.stopTicker() stopStream() @@ -270,6 +267,10 @@ onBeforeUnmount(() => { v-if="hasPermissions" :class="['flex', 'flex-col', 'gap-6']" > +
+ {{ refetchSources() }} +
+
@@ -486,7 +487,7 @@ onBeforeUnmount(() => { { />
- Vision input max size: {{ captureInputBounds.maxWidth }} x {{ captureInputBounds.maxHeight }} + Vision input max size: {{ captureInputBounds.maxWidth }} × {{ captureInputBounds.maxHeight }}
{
Snapshot - {{ captureCount }} captures, {{ contextUpdateCount }} context updates + {{ captureCount }} captures · {{ contextUpdateCount }} context updates
{ contextId: 'vision:screen:understand', }) }) - - it('records inference failures on the store before rethrowing', async () => { - const store = useVisionOrchestratorStore() - runVisionInference.mockRejectedValueOnce(new Error('Vision inference failed')) - - await expect(store.processCapture({ - imageDataUrl: 'data:image/jpeg;base64,broken', - workloadId: 'screen:interpret', - })).rejects.toThrow('Vision inference failed') - - expect(store.lastError).toBe('Vision inference failed') - }) }) diff --git a/packages/stage-ui/src/stores/modules/vision/orchestrator.ts b/packages/stage-ui/src/stores/modules/vision/orchestrator.ts index 64fcc380d..174b4ffe0 100644 --- a/packages/stage-ui/src/stores/modules/vision/orchestrator.ts +++ b/packages/stage-ui/src/stores/modules/vision/orchestrator.ts @@ -2,7 +2,6 @@ import type { CommonContentPart } from '@xsai/shared-chat' import type { VisionWorkloadId } from '../../../composables/vision/use-vision-workloads' -import { errorMessageFrom } from '@moeru/std' import { ContextUpdateStrategy } from '@proj-airi/server-sdk' import { defineStore, storeToRefs } from 'pinia' import { ref } from 'vue' @@ -12,19 +11,11 @@ import { getVisionWorkload } from '../../../composables/vision/use-vision-worklo import { useModsServerChannelStore } from '../../mods/api/channel-server' import { useVisionStore } from './store' -/** - * Payload describing one captured frame routed through the vision orchestrator. - */ export interface VisionCapturePayload { - /** JPEG or PNG data URL captured from the selected source. */ imageDataUrl: string - /** Vision workload that describes how the frame should be interpreted. */ workloadId: VisionWorkloadId - /** Optional source identifier used to keep context updates stable per source. */ sourceId?: string - /** Timestamp recorded when the frame was captured. */ capturedAt?: number - /** When `true`, publish the inference result into the character context channel. */ publishContext?: boolean } @@ -34,19 +25,6 @@ function getVisionContextId(payload: Pick { const visionStore = useVisionStore() const { activeProvider, activeModel } = storeToRefs(visionStore) @@ -59,64 +37,55 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', () const lastWorkloadId = ref('screen:interpret') async function processCapture(payload: VisionCapturePayload) { - if (!activeProvider.value || !activeModel.value) { - const configurationError = new Error('Vision model is not configured') - recordError(configurationError) - throw configurationError - } + if (!activeProvider.value || !activeModel.value) + throw new Error('Vision model is not configured') lastWorkloadId.value = payload.workloadId - try { - const text = await runVisionInference({ - imageDataUrl: payload.imageDataUrl, - workloadId: payload.workloadId, + const text = await runVisionInference({ + imageDataUrl: payload.imageDataUrl, + workloadId: payload.workloadId, + }) + + lastResultText.value = text + lastResultAt.value = Date.now() + lastError.value = null + + if (payload.publishContext) { + const workload = getVisionWorkload(payload.workloadId) + const content: CommonContentPart[] = [ + { type: 'text', text }, + { + type: 'image_url', + image_url: { + url: payload.imageDataUrl, + }, + }, + ] + + modsServerChannelStore.sendContextUpdate({ + strategy: ContextUpdateStrategy.ReplaceSelf, + contextId: getVisionContextId(payload), + text, + content, + metadata: { + module: 'vision', + workload: workload.id, + workloadLabel: workload.label, + sourceId: payload.sourceId, + capturedAt: payload.capturedAt, + provider: activeProvider.value, + model: activeModel.value, + }, }) - - lastResultText.value = text - lastResultAt.value = Date.now() - lastError.value = null - - if (payload.publishContext) { - const workload = getVisionWorkload(payload.workloadId) - const content: CommonContentPart[] = [ - { type: 'text', text }, - { - type: 'image_url', - image_url: { - url: payload.imageDataUrl, - }, - }, - ] - - modsServerChannelStore.sendContextUpdate({ - strategy: ContextUpdateStrategy.ReplaceSelf, - contextId: getVisionContextId(payload), - text, - content, - metadata: { - module: 'vision', - workload: workload.id, - workloadLabel: workload.label, - sourceId: payload.sourceId, - capturedAt: payload.capturedAt, - provider: activeProvider.value, - model: activeModel.value, - }, - }) - return { contextUpdates: 1, text } - } - - return { contextUpdates: 0, text } - } - catch (error) { - recordError(error) - throw error + return { contextUpdates: 1, text } } + + return { contextUpdates: 0, text } } function recordError(error: unknown) { - lastError.value = errorMessageFrom(error) ?? 'Unknown error' + lastError.value = error instanceof Error ? error.message : String(error) } return {