nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 5 Packages Projects Releases Wiki Activity

Revert "fix(stage-ui): correct stream reuse when switching sources + improve capture pipeline" (#1741)

Browse Source 
Reverts moeru-ai/airi#1731
This commit is contained in:
Neko
2026-04-27 15:22:19 +08:00
committed by GitHub
parent 4ba5b13c0d
commit 0c8c99dae3
7 changed files with 127 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts
+52 -66
View File
@@ -3,46 +3,35 @@ import type { SourcesOptions } from 'electron'
import type { MaybeRefOrGetter } from 'vue'
import { useElectronScreenCapture } from '@proj-airi/electron-screen-capture/vue'
import { computed, ref, shallowRef, watch } from 'vue'
import { createObjectUrlFromBytes } from '../utils/create-object-url-from-bytes'
import { computed, ref, toRaw, toValue } from 'vue'
interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
appIconURL?: string
thumbnailURL?: string
}
/**
* Manages Electron-backed screen-capture sources and the active preview stream for vision workflows.
*
* Use when:
* - A renderer page needs to browse screen/window sources before capturing frames
* - The page should keep a single active `MediaStream` in sync with the selected source
*
* Expects:
* - The Electron screen-capture preload APIs to be available on `window.electron.ipcRenderer`
* - Callers to invoke `cleanup()` when the owning component unmounts
*
* Returns:
* - Reactive source lists, active stream state, and helpers for refetching, starting, stopping, and capturing frames
*/
function toLocalArrayBuffer(bytes: Uint8Array) {
if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) {
return bytes.slice().buffer
}
return bytes.buffer as ArrayBuffer
}
function toObjectUrl(bytes: Uint8Array, mime: string) {
return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime }))
}
export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesOptions>) {
const sources = ref<ScreenCaptureSource[]>([])
const isRefetching = ref(false)
const hasFetchedOnce = ref(false)
const activeSourceId = ref('')
const activeStream = shallowRef<MediaStream | null>(null)
const activeStreamSourceId = ref('')
watch(activeSourceId, (nextId) => {
if (activeStreamSourceId.value && activeStreamSourceId.value !== nextId) {
clearActiveStream()
}
})
const activeStream = ref<MediaStream | null>(null)
const {
getSources,
selectWithSource,
setSource,
resetSource,
} = useElectronScreenCapture(window.electron.ipcRenderer, sourcesOptions)
const activeSource = computed(() => sources.value.find(source => source.id === activeSourceId.value) || null)
@@ -58,31 +47,18 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
const stream = activeStream.value
if (!stream) {
activeStream.value = null
activeStreamSourceId.value = ''
return
}
stream.getTracks().forEach(track => track.stop())
activeStream.value = null
activeStreamSourceId.value = ''
}
function revokeSourceObjectUrls(entries: ScreenCaptureSource[]) {
entries.forEach((source) => {
if (source.appIconURL)
URL.revokeObjectURL(source.appIconURL)
if (source.thumbnailURL)
URL.revokeObjectURL(source.thumbnailURL)
})
}
function attachStreamLifecycle(stream: MediaStream, sourceId: string) {
function attachStreamLifecycle(stream: MediaStream) {
stream.getTracks().forEach((track) => {
track.addEventListener('ended', () => {
if (activeStream.value === stream && activeStreamSourceId.value === sourceId) {
if (activeStream.value === stream)
activeStream.value = null
activeStreamSourceId.value = ''
}
}, { once: true })
})
}
@@ -99,17 +75,22 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
return a.name.localeCompare(b.name)
})
revokeSourceObjectUrls(sources.value)
sources.value.forEach((oldSource) => {
if (oldSource.appIconURL)
URL.revokeObjectURL(oldSource.appIconURL)
if (oldSource.thumbnailURL)
URL.revokeObjectURL(oldSource.thumbnailURL)
})
sources.value = nextSources.map(source => ({
...source,
appIconURL: source.appIcon && source.appIcon.length > 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined,
appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
}))
const hasActiveSource = sources.value.some(source => source.id === activeSourceId.value)
const nextActiveSourceId = hasActiveSource ? activeSourceId.value : sources.value[0]?.id || ''
activeSourceId.value = nextActiveSourceId
if (!hasActiveSource)
activeSourceId.value = sources.value[0]?.id || ''
}
finally {
isRefetching.value = false
@@ -118,29 +99,32 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
}
async function startStream() {
const sourceId = activeSourceId.value
if (!sourceId)
if (!activeSourceId.value)
throw new Error('No active source selected')
if (isActiveStream(activeStream.value) && activeStreamSourceId.value === sourceId)
if (isActiveStream(activeStream.value))
return activeStream.value!
clearActiveStream()
const stream = await selectWithSource(
() => sourceId,
async () => await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }),
)
if (!isActiveStream(stream)) {
stream.getTracks().forEach(track => track.stop())
throw new Error('Selected source did not provide a live video track')
const handle = await setSource({
options: toRaw(toValue(sourcesOptions)),
sourceId: activeSourceId.value,
})
try {
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false })
activeStream.value = stream
attachStreamLifecycle(stream)
return stream
}
catch (error) {
activeStream.value = null
throw error
}
finally {
await resetSource(handle)
}
activeStream.value = stream
activeStreamSourceId.value = sourceId
attachStreamLifecycle(stream, sourceId)
return stream
}
function stopStream() {
@@ -149,7 +133,12 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
function cleanup() {
stopStream()
revokeSourceObjectUrls(sources.value)
sources.value.forEach((oldSource) => {
if (oldSource.appIconURL)
URL.revokeObjectURL(oldSource.appIconURL)
if (oldSource.thumbnailURL)
URL.revokeObjectURL(oldSource.thumbnailURL)
})
}
function captureFrame(video: HTMLVideoElement, quality = 0.82, maxWidth = 1280, maxHeight = 720) {
@@ -159,9 +148,6 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
const canvas = document.createElement('canvas')
const sourceWidth = video.videoWidth
const sourceHeight = video.videoHeight
if (sourceWidth <= 0 || sourceHeight <= 0)
return null
const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1)
canvas.width = Math.round(sourceWidth * scale)
canvas.height = Math.round(sourceHeight * scale)
apps/stage-tamagotchi/src/renderer/pages/devtools/screen-capture.vue
+14 -5
View File
@@ -9,8 +9,6 @@ import { useI18n } from 'vue-i18n'
import WithScreenCapture from '../../components/WithScreenCapture.vue'
import { createObjectUrlFromBytes } from '../../utils/create-object-url-from-bytes'
interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
appIconURL?: string
thumbnailURL?: string
@@ -76,6 +74,17 @@ function getShareLabel(source: ScreenCaptureSource) {
return 'Share Window'
}
function toLocalArrayBuffer(bytes: Uint8Array) {
if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) {
return bytes.slice().buffer
}
return bytes.buffer as ArrayBuffer
}
function toObjectUrl(bytes: Uint8Array, mime: string) {
return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime }))
}
async function startCapture(source: SerializableDesktopCapturerSource) {
try {
await selectWithSource(
@@ -130,8 +139,8 @@ async function refetchSources() {
// NOTICE(@sumimakito): Not only thumbnail is empty, the appIcon could be empty as well with nothing returned.
// REVIEW(@sumimakito): This has nothing to do with our side, probably related to a Electron bug, you can
// read more here https://github.com/electron/electron/issues/44504
appIconURL: source.appIcon && source.appIcon.length > 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined,
appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
}))
}
catch (err) {
@@ -144,7 +153,7 @@ async function refetchSources() {
}
onMounted(async () => {
await refetchSources()
refetchSources()
})
onBeforeUnmount(() => {
apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue
+19 -18
View File
@@ -8,13 +8,13 @@ import { VISION_WORKLOADS } from '@proj-airi/stage-ui/composables'
import { useVisionOrchestratorStore, useVisionProcessingStore, useVisionStore } from '@proj-airi/stage-ui/stores/modules/vision'
import { Button, FieldCheckbox, FieldCombobox, FieldRange, SelectTab } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
import { computed, onBeforeUnmount, ref } from 'vue'
import WithScreenCapture from '../../components/WithScreenCapture.vue'
import { useVisionScreenCapture } from '../../composables/use-vision-screen-capture'
type SourceCategory = 'applications' | 'displays'
type SourceCategory = 'applications' | 'displays' | 'devices'
const visionStore = useVisionStore()
const visionProcessingStore = useVisionProcessingStore()
@@ -68,6 +68,7 @@ const {
const categoryOptions = [
{ label: 'Applications', value: 'applications', icon: 'i-solar:window-frame-line-duotone' },
{ label: 'Displays', value: 'displays', icon: 'i-solar:screencast-2-line-duotone' },
{ label: 'Devices', value: 'devices', icon: 'i-solar:smartphone-2-line-duotone' },
]
const workloadOptions = VISION_WORKLOADS.map(workload => ({
@@ -77,21 +78,27 @@ const workloadOptions = VISION_WORKLOADS.map(workload => ({
const isDisplaySource = (source: { id: string }) => source.id.startsWith('screen:')
const isWindowSource = (source: { id: string }) => source.id.startsWith('window:')
const isDeviceSource = (source: { id: string }) => source.id.startsWith('device:')
const filteredSources = computed(() => {
if (sourceCategory.value === 'applications')
return sources.value.filter(isWindowSource)
return sources.value.filter(isDisplaySource)
if (sourceCategory.value === 'displays')
return sources.value.filter(isDisplaySource)
return sources.value.filter(isDeviceSource)
})
const sourceCounts = computed(() => ({
applications: sources.value.filter(isWindowSource).length,
displays: sources.value.filter(isDisplaySource).length,
devices: sources.value.filter(isDeviceSource).length,
}))
function getShareLabel(source: { id: string }) {
if (isDisplaySource(source))
return 'Share Screen'
if (isDeviceSource(source))
return 'Share Device'
return 'Share Window'
}
@@ -147,13 +154,7 @@ async function ensureVideoStream() {
resolve()
return
}
const handleLoadedMetadata = () => {
video.removeEventListener('loadedmetadata', handleLoadedMetadata)
resolve()
}
video.addEventListener('loadedmetadata', handleLoadedMetadata)
video.onloadedmetadata = () => resolve()
})
}
@@ -163,7 +164,7 @@ async function handleVisionTick() {
try {
if (!hasLiveVideoStream(activeStream.value)) {
stopStream()
activeStream.value = null
await ensureVideoStream()
}
@@ -252,10 +253,6 @@ async function shareSource(sourceId: string) {
}
}
onMounted(() => {
void refetchSources()
})
onBeforeUnmount(() => {
visionProcessingStore.stopTicker()
stopStream()
@@ -270,6 +267,10 @@ onBeforeUnmount(() => {
v-if="hasPermissions"
:class="['flex', 'flex-col', 'gap-6']"
>
<div v-if="!hasFetchedOnce && !isRefetching" :class="['hidden']">
{{ refetchSources() }}
</div>
<div :class="['flex', 'items-center', 'justify-between', 'rounded-xl', 'bg-neutral-100', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
<div :class="['flex', 'flex-col', 'gap-1']">
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
@@ -486,7 +487,7 @@ onBeforeUnmount(() => {
<FieldRange
v-model="captureDownscalePercent"
label="Input downscale"
description="Shrink each captured frame before sending it to the vision model. 100% keeps the existing 1280x720 capture cap."
description="Shrink each captured frame before sending it to the vision model. 100% keeps the existing 1280×720 capture cap."
:min="25"
:max="100"
:step="5"
@@ -494,7 +495,7 @@ onBeforeUnmount(() => {
/>
<div :class="['text-xs', 'text-neutral-400']">
Vision input max size: {{ captureInputBounds.maxWidth }} x {{ captureInputBounds.maxHeight }}
Vision input max size: {{ captureInputBounds.maxWidth }} × {{ captureInputBounds.maxHeight }}
</div>
<FieldCombobox
@@ -548,7 +549,7 @@ onBeforeUnmount(() => {
<div :class="['rounded-xl', 'bg-neutral-100', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
<div :class="['flex', 'items-center', 'justify-between', 'text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
<span>Snapshot</span>
<span>{{ captureCount }} captures, {{ contextUpdateCount }} context updates</span>
<span>{{ captureCount }} captures · {{ contextUpdateCount }} context updates</span>
</div>
<div
v-if="screenshotDataUrl"
apps/stage-tamagotchi/src/renderer/utils/create-object-url-from-bytes.ts
-17
View File
@@ -1,17 +0,0 @@
/**
* Creates a blob URL from a byte view without leaking unrelated backing-buffer bytes.
*
* Use when:
* - Electron screen-capture APIs return `Uint8Array` thumbnails or app icons
* - The byte view may point at a sliced `ArrayBuffer` or `SharedArrayBuffer`
*
* Expects:
* - `bytes` to contain only the payload that should be exposed through the blob URL
*
* Returns:
* - A `blob:` URL that the caller must revoke when it is no longer needed
*/
export function createObjectUrlFromBytes(bytes: Uint8Array, mime: string): string {
const ownedBytes = Uint8Array.from(bytes)
return URL.createObjectURL(new Blob([ownedBytes], { type: mime }))
}
nix/assets-hash.txt
+1 -1
View File
@@ -1 +1 @@
sha256-xa45blGXsW12Fh2ivSPrB8mR7+ui9WJ6ki0Zj73GMa8=
sha256-+ruQJso6gF5n4vdu9xTMiYoTP6q3xRrCVsGrEEggQqc=
packages/stage-ui/src/stores/modules/vision/orchestrator.test.ts
-12
View File
@@ -93,16 +93,4 @@ describe('vision orchestrator', () => {
contextId: 'vision:screen:understand',
})
})
it('records inference failures on the store before rethrowing', async () => {
const store = useVisionOrchestratorStore()
runVisionInference.mockRejectedValueOnce(new Error('Vision inference failed'))
await expect(store.processCapture({
imageDataUrl: 'data:image/jpeg;base64,broken',
workloadId: 'screen:interpret',
})).rejects.toThrow('Vision inference failed')
expect(store.lastError).toBe('Vision inference failed')
})
})
packages/stage-ui/src/stores/modules/vision/orchestrator.ts
+41 -72
View File
@@ -2,7 +2,6 @@ import type { CommonContentPart } from '@xsai/shared-chat'
import type { VisionWorkloadId } from '../../../composables/vision/use-vision-workloads'
import { errorMessageFrom } from '@moeru/std'
import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
import { defineStore, storeToRefs } from 'pinia'
import { ref } from 'vue'
@@ -12,19 +11,11 @@ import { getVisionWorkload } from '../../../composables/vision/use-vision-worklo
import { useModsServerChannelStore } from '../../mods/api/channel-server'
import { useVisionStore } from './store'
/**
* Payload describing one captured frame routed through the vision orchestrator.
*/
export interface VisionCapturePayload {
/** JPEG or PNG data URL captured from the selected source. */
imageDataUrl: string
/** Vision workload that describes how the frame should be interpreted. */
workloadId: VisionWorkloadId
/** Optional source identifier used to keep context updates stable per source. */
sourceId?: string
/** Timestamp recorded when the frame was captured. */
capturedAt?: number
/** When `true`, publish the inference result into the character context channel. */
publishContext?: boolean
}
@@ -34,19 +25,6 @@ function getVisionContextId(payload: Pick<VisionCapturePayload, 'workloadId' | '
: `vision:${payload.workloadId}`
}
/**
* Coordinates screen-capture inference and optional context publishing for vision workflows.
*
* Use when:
* - A renderer page captures frames and needs multimodal inference results
* - Successful results may also need to become context updates for downstream modules
*
* Expects:
* - The vision settings store to already contain an active provider and model
*
* Returns:
* - A Pinia store that tracks the latest result, last error, and capture-processing actions
*/
export const useVisionOrchestratorStore = defineStore('vision-orchestrator', () => {
const visionStore = useVisionStore()
const { activeProvider, activeModel } = storeToRefs(visionStore)
@@ -59,64 +37,55 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', ()
const lastWorkloadId = ref<VisionWorkloadId>('screen:interpret')
async function processCapture(payload: VisionCapturePayload) {
if (!activeProvider.value || !activeModel.value) {
const configurationError = new Error('Vision model is not configured')
recordError(configurationError)
throw configurationError
}
if (!activeProvider.value || !activeModel.value)
throw new Error('Vision model is not configured')
lastWorkloadId.value = payload.workloadId
try {
const text = await runVisionInference({
imageDataUrl: payload.imageDataUrl,
workloadId: payload.workloadId,
const text = await runVisionInference({
imageDataUrl: payload.imageDataUrl,
workloadId: payload.workloadId,
})
lastResultText.value = text
lastResultAt.value = Date.now()
lastError.value = null
if (payload.publishContext) {
const workload = getVisionWorkload(payload.workloadId)
const content: CommonContentPart[] = [
{ type: 'text', text },
{
type: 'image_url',
image_url: {
url: payload.imageDataUrl,
},
},
]
modsServerChannelStore.sendContextUpdate({
strategy: ContextUpdateStrategy.ReplaceSelf,
contextId: getVisionContextId(payload),
text,
content,
metadata: {
module: 'vision',
workload: workload.id,
workloadLabel: workload.label,
sourceId: payload.sourceId,
capturedAt: payload.capturedAt,
provider: activeProvider.value,
model: activeModel.value,
},
})
lastResultText.value = text
lastResultAt.value = Date.now()
lastError.value = null
if (payload.publishContext) {
const workload = getVisionWorkload(payload.workloadId)
const content: CommonContentPart[] = [
{ type: 'text', text },
{
type: 'image_url',
image_url: {
url: payload.imageDataUrl,
},
},
]
modsServerChannelStore.sendContextUpdate({
strategy: ContextUpdateStrategy.ReplaceSelf,
contextId: getVisionContextId(payload),
text,
content,
metadata: {
module: 'vision',
workload: workload.id,
workloadLabel: workload.label,
sourceId: payload.sourceId,
capturedAt: payload.capturedAt,
provider: activeProvider.value,
model: activeModel.value,
},
})
return { contextUpdates: 1, text }
}
return { contextUpdates: 0, text }
}
catch (error) {
recordError(error)
throw error
return { contextUpdates: 1, text }
}
return { contextUpdates: 0, text }
}
function recordError(error: unknown) {
lastError.value = errorMessageFrom(error) ?? 'Unknown error'
lastError.value = error instanceof Error ? error.message : String(error)
}
return {