Revert "fix(stage-ui): correct stream reuse when switching sources + improve capture pipeline" (#1741)

Reverts moeru-ai/airi#1731

apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts

@@ -3,46 +3,35 @@ import type { SourcesOptions } from 'electron'
 import type { MaybeRefOrGetter } from 'vue'
 
 import { useElectronScreenCapture } from '@proj-airi/electron-screen-capture/vue'
-import { computed, ref, shallowRef, watch } from 'vue'
+import { computed, ref, toRaw, toValue } from 'vue'
-
-import { createObjectUrlFromBytes } from '../utils/create-object-url-from-bytes'
 
 interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
   appIconURL?: string
   thumbnailURL?: string
 }
 
-/**
+function toLocalArrayBuffer(bytes: Uint8Array) {
- * Manages Electron-backed screen-capture sources and the active preview stream for vision workflows.
+  if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) {
- *
+    return bytes.slice().buffer
- * Use when:
+  }
- * - A renderer page needs to browse screen/window sources before capturing frames
+  return bytes.buffer as ArrayBuffer
- * - The page should keep a single active `MediaStream` in sync with the selected source
+}
- *
+
- * Expects:
+function toObjectUrl(bytes: Uint8Array, mime: string) {
- * - The Electron screen-capture preload APIs to be available on `window.electron.ipcRenderer`
+  return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime }))
- * - Callers to invoke `cleanup()` when the owning component unmounts
+}
- *
+
- * Returns:
- * - Reactive source lists, active stream state, and helpers for refetching, starting, stopping, and capturing frames
- */
 export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesOptions>) {
   const sources = ref<ScreenCaptureSource[]>([])
   const isRefetching = ref(false)
   const hasFetchedOnce = ref(false)
   const activeSourceId = ref('')
-  const activeStream = shallowRef<MediaStream | null>(null)
+  const activeStream = ref<MediaStream | null>(null)
-  const activeStreamSourceId = ref('')
-
-  watch(activeSourceId, (nextId) => {
-    if (activeStreamSourceId.value && activeStreamSourceId.value !== nextId) {
-      clearActiveStream()
-    }
-  })
 
   const {
     getSources,
-    selectWithSource,
+    setSource,
+    resetSource,
   } = useElectronScreenCapture(window.electron.ipcRenderer, sourcesOptions)
 
   const activeSource = computed(() => sources.value.find(source => source.id === activeSourceId.value) || null)
@@ -58,31 +47,18 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
     const stream = activeStream.value
     if (!stream) {
       activeStream.value = null
-      activeStreamSourceId.value = ''
       return
     }
 
     stream.getTracks().forEach(track => track.stop())
     activeStream.value = null
-    activeStreamSourceId.value = ''
   }
 
-  function revokeSourceObjectUrls(entries: ScreenCaptureSource[]) {
+  function attachStreamLifecycle(stream: MediaStream) {
-    entries.forEach((source) => {
-      if (source.appIconURL)
-        URL.revokeObjectURL(source.appIconURL)
-      if (source.thumbnailURL)
-        URL.revokeObjectURL(source.thumbnailURL)
-    })
-  }
-
-  function attachStreamLifecycle(stream: MediaStream, sourceId: string) {
     stream.getTracks().forEach((track) => {
       track.addEventListener('ended', () => {
-        if (activeStream.value === stream && activeStreamSourceId.value === sourceId) {
+        if (activeStream.value === stream)
           activeStream.value = null
-          activeStreamSourceId.value = ''
-        }
       }, { once: true })
     })
   }
@@ -99,17 +75,22 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
           return a.name.localeCompare(b.name)
         })
 
-      revokeSourceObjectUrls(sources.value)
+      sources.value.forEach((oldSource) => {
+        if (oldSource.appIconURL)
+          URL.revokeObjectURL(oldSource.appIconURL)
+        if (oldSource.thumbnailURL)
+          URL.revokeObjectURL(oldSource.thumbnailURL)
+      })
 
       sources.value = nextSources.map(source => ({
         ...source,
-        appIconURL: source.appIcon && source.appIcon.length > 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined,
+        appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
-        thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined,
+        thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
       }))
 
       const hasActiveSource = sources.value.some(source => source.id === activeSourceId.value)
-      const nextActiveSourceId = hasActiveSource ? activeSourceId.value : sources.value[0]?.id || ''
+      if (!hasActiveSource)
-      activeSourceId.value = nextActiveSourceId
+        activeSourceId.value = sources.value[0]?.id || ''
     }
     finally {
       isRefetching.value = false
@@ -118,29 +99,32 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
   }
 
   async function startStream() {
-    const sourceId = activeSourceId.value
+    if (!activeSourceId.value)
-    if (!sourceId)
       throw new Error('No active source selected')
 
-    if (isActiveStream(activeStream.value) && activeStreamSourceId.value === sourceId)
+    if (isActiveStream(activeStream.value))
       return activeStream.value!
 
     clearActiveStream()
 
-    const stream = await selectWithSource(
+    const handle = await setSource({
-      () => sourceId,
+      options: toRaw(toValue(sourcesOptions)),
-      async () => await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }),
+      sourceId: activeSourceId.value,
-    )
+    })
-    if (!isActiveStream(stream)) {
+
-      stream.getTracks().forEach(track => track.stop())
+    try {
-      throw new Error('Selected source did not provide a live video track')
+      const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false })
+      activeStream.value = stream
+      attachStreamLifecycle(stream)
+      return stream
+    }
+    catch (error) {
+      activeStream.value = null
+      throw error
+    }
+    finally {
+      await resetSource(handle)
     }
-
-    activeStream.value = stream
-    activeStreamSourceId.value = sourceId
-    attachStreamLifecycle(stream, sourceId)
-
-    return stream
   }
 
   function stopStream() {
@@ -149,7 +133,12 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
 
   function cleanup() {
     stopStream()
-    revokeSourceObjectUrls(sources.value)
+    sources.value.forEach((oldSource) => {
+      if (oldSource.appIconURL)
+        URL.revokeObjectURL(oldSource.appIconURL)
+      if (oldSource.thumbnailURL)
+        URL.revokeObjectURL(oldSource.thumbnailURL)
+    })
   }
 
   function captureFrame(video: HTMLVideoElement, quality = 0.82, maxWidth = 1280, maxHeight = 720) {
@@ -159,9 +148,6 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
     const canvas = document.createElement('canvas')
     const sourceWidth = video.videoWidth
     const sourceHeight = video.videoHeight
-    if (sourceWidth <= 0 || sourceHeight <= 0)
-      return null
-
     const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1)
     canvas.width = Math.round(sourceWidth * scale)
     canvas.height = Math.round(sourceHeight * scale)

apps/stage-tamagotchi/src/renderer/pages/devtools/screen-capture.vue

@@ -9,8 +9,6 @@ import { useI18n } from 'vue-i18n'
 
 import WithScreenCapture from '../../components/WithScreenCapture.vue'
 
-import { createObjectUrlFromBytes } from '../../utils/create-object-url-from-bytes'
-
 interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
   appIconURL?: string
   thumbnailURL?: string
@@ -76,6 +74,17 @@ function getShareLabel(source: ScreenCaptureSource) {
   return 'Share Window'
 }
 
+function toLocalArrayBuffer(bytes: Uint8Array) {
+  if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) {
+    return bytes.slice().buffer
+  }
+  return bytes.buffer as ArrayBuffer
+}
+
+function toObjectUrl(bytes: Uint8Array, mime: string) {
+  return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime }))
+}
+
 async function startCapture(source: SerializableDesktopCapturerSource) {
   try {
     await selectWithSource(
@@ -130,8 +139,8 @@ async function refetchSources() {
       // NOTICE(@sumimakito): Not only thumbnail is empty, the appIcon could be empty as well with nothing returned.
       // REVIEW(@sumimakito): This has nothing to do with our side, probably related to a Electron bug, you can
       // read more here https://github.com/electron/electron/issues/44504
-      appIconURL: source.appIcon && source.appIcon.length > 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined,
+      appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
-      thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined,
+      thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
     }))
   }
   catch (err) {
@@ -144,7 +153,7 @@ async function refetchSources() {
 }
 
 onMounted(async () => {
-  await refetchSources()
+  refetchSources()
 })
 
 onBeforeUnmount(() => {

apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue

@@ -8,13 +8,13 @@ import { VISION_WORKLOADS } from '@proj-airi/stage-ui/composables'
 import { useVisionOrchestratorStore, useVisionProcessingStore, useVisionStore } from '@proj-airi/stage-ui/stores/modules/vision'
 import { Button, FieldCheckbox, FieldCombobox, FieldRange, SelectTab } from '@proj-airi/ui'
 import { storeToRefs } from 'pinia'
-import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
+import { computed, onBeforeUnmount, ref } from 'vue'
 
 import WithScreenCapture from '../../components/WithScreenCapture.vue'
 
 import { useVisionScreenCapture } from '../../composables/use-vision-screen-capture'
 
-type SourceCategory = 'applications' | 'displays'
+type SourceCategory = 'applications' | 'displays' | 'devices'
 
 const visionStore = useVisionStore()
 const visionProcessingStore = useVisionProcessingStore()
@@ -68,6 +68,7 @@ const {
 const categoryOptions = [
   { label: 'Applications', value: 'applications', icon: 'i-solar:window-frame-line-duotone' },
   { label: 'Displays', value: 'displays', icon: 'i-solar:screencast-2-line-duotone' },
+  { label: 'Devices', value: 'devices', icon: 'i-solar:smartphone-2-line-duotone' },
 ]
 
 const workloadOptions = VISION_WORKLOADS.map(workload => ({
@@ -77,21 +78,27 @@ const workloadOptions = VISION_WORKLOADS.map(workload => ({
 
 const isDisplaySource = (source: { id: string }) => source.id.startsWith('screen:')
 const isWindowSource = (source: { id: string }) => source.id.startsWith('window:')
+const isDeviceSource = (source: { id: string }) => source.id.startsWith('device:')
 
 const filteredSources = computed(() => {
   if (sourceCategory.value === 'applications')
     return sources.value.filter(isWindowSource)
-  return sources.value.filter(isDisplaySource)
+  if (sourceCategory.value === 'displays')
+    return sources.value.filter(isDisplaySource)
+  return sources.value.filter(isDeviceSource)
 })
 
 const sourceCounts = computed(() => ({
   applications: sources.value.filter(isWindowSource).length,
   displays: sources.value.filter(isDisplaySource).length,
+  devices: sources.value.filter(isDeviceSource).length,
 }))
 
 function getShareLabel(source: { id: string }) {
   if (isDisplaySource(source))
     return 'Share Screen'
+  if (isDeviceSource(source))
+    return 'Share Device'
   return 'Share Window'
 }
 
@@ -147,13 +154,7 @@ async function ensureVideoStream() {
       resolve()
       return
     }
-
+    video.onloadedmetadata = () => resolve()
-    const handleLoadedMetadata = () => {
-      video.removeEventListener('loadedmetadata', handleLoadedMetadata)
-      resolve()
-    }
-
-    video.addEventListener('loadedmetadata', handleLoadedMetadata)
   })
 }
 
@@ -163,7 +164,7 @@ async function handleVisionTick() {
 
   try {
     if (!hasLiveVideoStream(activeStream.value)) {
-      stopStream()
+      activeStream.value = null
       await ensureVideoStream()
     }
 
@@ -252,10 +253,6 @@ async function shareSource(sourceId: string) {
   }
 }
 
-onMounted(() => {
-  void refetchSources()
-})
-
 onBeforeUnmount(() => {
   visionProcessingStore.stopTicker()
   stopStream()
@@ -270,6 +267,10 @@ onBeforeUnmount(() => {
         v-if="hasPermissions"
         :class="['flex', 'flex-col', 'gap-6']"
       >
+        <div v-if="!hasFetchedOnce && !isRefetching" :class="['hidden']">
+          {{ refetchSources() }}
+        </div>
+
         <div :class="['flex', 'items-center', 'justify-between', 'rounded-xl', 'bg-neutral-100', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
           <div :class="['flex', 'flex-col', 'gap-1']">
             <div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
@@ -486,7 +487,7 @@ onBeforeUnmount(() => {
                 <FieldRange
                   v-model="captureDownscalePercent"
                   label="Input downscale"
-                  description="Shrink each captured frame before sending it to the vision model. 100% keeps the existing 1280x720 capture cap."
+                  description="Shrink each captured frame before sending it to the vision model. 100% keeps the existing 1280×720 capture cap."
                   :min="25"
                   :max="100"
                   :step="5"
@@ -494,7 +495,7 @@ onBeforeUnmount(() => {
                 />
 
                 <div :class="['text-xs', 'text-neutral-400']">
-                  Vision input max size: {{ captureInputBounds.maxWidth }} x {{ captureInputBounds.maxHeight }}
+                  Vision input max size: {{ captureInputBounds.maxWidth }} × {{ captureInputBounds.maxHeight }}
                 </div>
 
                 <FieldCombobox
@@ -548,7 +549,7 @@ onBeforeUnmount(() => {
             <div :class="['rounded-xl', 'bg-neutral-100', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
               <div :class="['flex', 'items-center', 'justify-between', 'text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
                 <span>Snapshot</span>
-                <span>{{ captureCount }} captures, {{ contextUpdateCount }} context updates</span>
+                <span>{{ captureCount }} captures · {{ contextUpdateCount }} context updates</span>
               </div>
               <div
                 v-if="screenshotDataUrl"

apps/stage-tamagotchi/src/renderer/utils/create-object-url-from-bytes.ts

@@ -1,17 +0,0 @@
-/**
- * Creates a blob URL from a byte view without leaking unrelated backing-buffer bytes.
- *
- * Use when:
- * - Electron screen-capture APIs return `Uint8Array` thumbnails or app icons
- * - The byte view may point at a sliced `ArrayBuffer` or `SharedArrayBuffer`
- *
- * Expects:
- * - `bytes` to contain only the payload that should be exposed through the blob URL
- *
- * Returns:
- * - A `blob:` URL that the caller must revoke when it is no longer needed
- */
-export function createObjectUrlFromBytes(bytes: Uint8Array, mime: string): string {
-  const ownedBytes = Uint8Array.from(bytes)
-  return URL.createObjectURL(new Blob([ownedBytes], { type: mime }))
-}

nix/assets-hash.txt

@@ -1 +1 @@
-sha256-xa45blGXsW12Fh2ivSPrB8mR7+ui9WJ6ki0Zj73GMa8=
+sha256-+ruQJso6gF5n4vdu9xTMiYoTP6q3xRrCVsGrEEggQqc=

packages/stage-ui/src/stores/modules/vision/orchestrator.test.ts

@@ -93,16 +93,4 @@ describe('vision orchestrator', () => {
       contextId: 'vision:screen:understand',
     })
   })
-
-  it('records inference failures on the store before rethrowing', async () => {
-    const store = useVisionOrchestratorStore()
-    runVisionInference.mockRejectedValueOnce(new Error('Vision inference failed'))
-
-    await expect(store.processCapture({
-      imageDataUrl: 'data:image/jpeg;base64,broken',
-      workloadId: 'screen:interpret',
-    })).rejects.toThrow('Vision inference failed')
-
-    expect(store.lastError).toBe('Vision inference failed')
-  })
 })

packages/stage-ui/src/stores/modules/vision/orchestrator.ts

@@ -2,7 +2,6 @@ import type { CommonContentPart } from '@xsai/shared-chat'
 
 import type { VisionWorkloadId } from '../../../composables/vision/use-vision-workloads'
 
-import { errorMessageFrom } from '@moeru/std'
 import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
 import { defineStore, storeToRefs } from 'pinia'
 import { ref } from 'vue'
@@ -12,19 +11,11 @@ import { getVisionWorkload } from '../../../composables/vision/use-vision-worklo
 import { useModsServerChannelStore } from '../../mods/api/channel-server'
 import { useVisionStore } from './store'
 
-/**
- * Payload describing one captured frame routed through the vision orchestrator.
- */
 export interface VisionCapturePayload {
-  /** JPEG or PNG data URL captured from the selected source. */
   imageDataUrl: string
-  /** Vision workload that describes how the frame should be interpreted. */
   workloadId: VisionWorkloadId
-  /** Optional source identifier used to keep context updates stable per source. */
   sourceId?: string
-  /** Timestamp recorded when the frame was captured. */
   capturedAt?: number
-  /** When `true`, publish the inference result into the character context channel. */
   publishContext?: boolean
 }
 
@@ -34,19 +25,6 @@ function getVisionContextId(payload: Pick<VisionCapturePayload, 'workloadId' | '
     : `vision:${payload.workloadId}`
 }
 
-/**
- * Coordinates screen-capture inference and optional context publishing for vision workflows.
- *
- * Use when:
- * - A renderer page captures frames and needs multimodal inference results
- * - Successful results may also need to become context updates for downstream modules
- *
- * Expects:
- * - The vision settings store to already contain an active provider and model
- *
- * Returns:
- * - A Pinia store that tracks the latest result, last error, and capture-processing actions
- */
 export const useVisionOrchestratorStore = defineStore('vision-orchestrator', () => {
   const visionStore = useVisionStore()
   const { activeProvider, activeModel } = storeToRefs(visionStore)
@@ -59,64 +37,55 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', ()
   const lastWorkloadId = ref<VisionWorkloadId>('screen:interpret')
 
   async function processCapture(payload: VisionCapturePayload) {
-    if (!activeProvider.value || !activeModel.value) {
+    if (!activeProvider.value || !activeModel.value)
-      const configurationError = new Error('Vision model is not configured')
+      throw new Error('Vision model is not configured')
-      recordError(configurationError)
-      throw configurationError
-    }
 
     lastWorkloadId.value = payload.workloadId
 
-    try {
+    const text = await runVisionInference({
-      const text = await runVisionInference({
+      imageDataUrl: payload.imageDataUrl,
-        imageDataUrl: payload.imageDataUrl,
+      workloadId: payload.workloadId,
-        workloadId: payload.workloadId,
+    })
+
+    lastResultText.value = text
+    lastResultAt.value = Date.now()
+    lastError.value = null
+
+    if (payload.publishContext) {
+      const workload = getVisionWorkload(payload.workloadId)
+      const content: CommonContentPart[] = [
+        { type: 'text', text },
+        {
+          type: 'image_url',
+          image_url: {
+            url: payload.imageDataUrl,
+          },
+        },
+      ]
+
+      modsServerChannelStore.sendContextUpdate({
+        strategy: ContextUpdateStrategy.ReplaceSelf,
+        contextId: getVisionContextId(payload),
+        text,
+        content,
+        metadata: {
+          module: 'vision',
+          workload: workload.id,
+          workloadLabel: workload.label,
+          sourceId: payload.sourceId,
+          capturedAt: payload.capturedAt,
+          provider: activeProvider.value,
+          model: activeModel.value,
+        },
       })
-
+      return { contextUpdates: 1, text }
-      lastResultText.value = text
-      lastResultAt.value = Date.now()
-      lastError.value = null
-
-      if (payload.publishContext) {
-        const workload = getVisionWorkload(payload.workloadId)
-        const content: CommonContentPart[] = [
-          { type: 'text', text },
-          {
-            type: 'image_url',
-            image_url: {
-              url: payload.imageDataUrl,
-            },
-          },
-        ]
-
-        modsServerChannelStore.sendContextUpdate({
-          strategy: ContextUpdateStrategy.ReplaceSelf,
-          contextId: getVisionContextId(payload),
-          text,
-          content,
-          metadata: {
-            module: 'vision',
-            workload: workload.id,
-            workloadLabel: workload.label,
-            sourceId: payload.sourceId,
-            capturedAt: payload.capturedAt,
-            provider: activeProvider.value,
-            model: activeModel.value,
-          },
-        })
-        return { contextUpdates: 1, text }
-      }
-
-      return { contextUpdates: 0, text }
-    }
-    catch (error) {
-      recordError(error)
-      throw error
     }
+
+    return { contextUpdates: 0, text }
   }
 
   function recordError(error: unknown) {
-    lastError.value = errorMessageFrom(error) ?? 'Unknown error'
+    lastError.value = error instanceof Error ? error.message : String(error)
   }
 
   return {