nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

fix(stage-ui): correct stream reuse when switching sources + improve capture pipeline (#1744)

Browse Source
This commit is contained in:
Iro
2026-04-28 14:50:59 +08:00
committed by GitHub
parent d413225346
commit cb7e058067
9 changed files with 203 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/stage-tamagotchi/src/renderer/components/WithScreenCapture.vue
+10
View File
@@ -12,6 +12,10 @@ const props = defineProps<{
sourcesOptions: SourcesOptions
}>()
const emit = defineEmits<{
permissionGranted: []
}>()
const sourcesOptions = toRef(props, 'sourcesOptions')
const hasPermissions = ref(false)
@@ -57,6 +61,12 @@ watch(focused, async (isFocused) => {
await checkPermissions()
}
})
watch(hasPermissions, (nextHasPermissions, previousHasPermissions) => {
if (nextHasPermissions && !previousHasPermissions) {
emit('permissionGranted')
}
})
</script>
<template>
apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts
+66 -52
View File
@@ -3,35 +3,46 @@ import type { SourcesOptions } from 'electron'
import type { MaybeRefOrGetter } from 'vue'
import { useElectronScreenCapture } from '@proj-airi/electron-screen-capture/vue'
import { computed, ref, toRaw, toValue } from 'vue'
import { computed, ref, shallowRef, watch } from 'vue'
import { createObjectUrlFromBytes } from '../utils/create-object-url-from-bytes'
interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
appIconURL?: string
thumbnailURL?: string
}
function toLocalArrayBuffer(bytes: Uint8Array) {
if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) {
return bytes.slice().buffer
}
return bytes.buffer as ArrayBuffer
}
function toObjectUrl(bytes: Uint8Array, mime: string) {
return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime }))
}
/**
* Manages Electron-backed screen-capture sources and the active preview stream for vision workflows.
*
* Use when:
* - A renderer page needs to browse screen/window sources before capturing frames
* - The page should keep a single active `MediaStream` in sync with the selected source
*
* Expects:
* - The Electron screen-capture preload APIs to be available on `window.electron.ipcRenderer`
* - Callers to invoke `cleanup()` when the owning component unmounts
*
* Returns:
* - Reactive source lists, active stream state, and helpers for refetching, starting, stopping, and capturing frames
*/
export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesOptions>) {
const sources = ref<ScreenCaptureSource[]>([])
const isRefetching = ref(false)
const hasFetchedOnce = ref(false)
const activeSourceId = ref('')
const activeStream = ref<MediaStream | null>(null)
const activeStream = shallowRef<MediaStream | null>(null)
const activeStreamSourceId = ref('')
watch(activeSourceId, (nextId) => {
if (activeStreamSourceId.value && activeStreamSourceId.value !== nextId) {
clearActiveStream()
}
})
const {
getSources,
setSource,
resetSource,
selectWithSource,
} = useElectronScreenCapture(window.electron.ipcRenderer, sourcesOptions)
const activeSource = computed(() => sources.value.find(source => source.id === activeSourceId.value) || null)
@@ -47,18 +58,31 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
const stream = activeStream.value
if (!stream) {
activeStream.value = null
activeStreamSourceId.value = ''
return
}
stream.getTracks().forEach(track => track.stop())
activeStream.value = null
activeStreamSourceId.value = ''
}
function attachStreamLifecycle(stream: MediaStream) {
function revokeSourceObjectUrls(entries: ScreenCaptureSource[]) {
entries.forEach((source) => {
if (source.appIconURL)
URL.revokeObjectURL(source.appIconURL)
if (source.thumbnailURL)
URL.revokeObjectURL(source.thumbnailURL)
})
}
function attachStreamLifecycle(stream: MediaStream, sourceId: string) {
stream.getTracks().forEach((track) => {
track.addEventListener('ended', () => {
if (activeStream.value === stream)
if (activeStream.value === stream && activeStreamSourceId.value === sourceId) {
activeStream.value = null
activeStreamSourceId.value = ''
}
}, { once: true })
})
}
@@ -75,22 +99,17 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
return a.name.localeCompare(b.name)
})
sources.value.forEach((oldSource) => {
if (oldSource.appIconURL)
URL.revokeObjectURL(oldSource.appIconURL)
if (oldSource.thumbnailURL)
URL.revokeObjectURL(oldSource.thumbnailURL)
})
revokeSourceObjectUrls(sources.value)
sources.value = nextSources.map(source => ({
...source,
appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
appIconURL: source.appIcon && source.appIcon.length > 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined,
}))
const hasActiveSource = sources.value.some(source => source.id === activeSourceId.value)
if (!hasActiveSource)
activeSourceId.value = sources.value[0]?.id || ''
const nextActiveSourceId = hasActiveSource ? activeSourceId.value : sources.value[0]?.id || ''
activeSourceId.value = nextActiveSourceId
}
finally {
isRefetching.value = false
@@ -99,32 +118,29 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
}
async function startStream() {
if (!activeSourceId.value)
const sourceId = activeSourceId.value
if (!sourceId)
throw new Error('No active source selected')
if (isActiveStream(activeStream.value))
if (isActiveStream(activeStream.value) && activeStreamSourceId.value === sourceId)
return activeStream.value!
clearActiveStream()
const handle = await setSource({
options: toRaw(toValue(sourcesOptions)),
sourceId: activeSourceId.value,
})
const stream = await selectWithSource(
() => sourceId,
async () => await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }),
)
if (!isActiveStream(stream)) {
stream.getTracks().forEach(track => track.stop())
throw new Error('Selected source did not provide a live video track')
}
try {
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false })
activeStream.value = stream
attachStreamLifecycle(stream)
return stream
}
catch (error) {
activeStream.value = null
throw error
}
finally {
await resetSource(handle)
}
activeStream.value = stream
activeStreamSourceId.value = sourceId
attachStreamLifecycle(stream, sourceId)
return stream
}
function stopStream() {
@@ -133,12 +149,7 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
function cleanup() {
stopStream()
sources.value.forEach((oldSource) => {
if (oldSource.appIconURL)
URL.revokeObjectURL(oldSource.appIconURL)
if (oldSource.thumbnailURL)
URL.revokeObjectURL(oldSource.thumbnailURL)
})
revokeSourceObjectUrls(sources.value)
}
function captureFrame(video: HTMLVideoElement, quality = 0.82, maxWidth = 1280, maxHeight = 720) {
@@ -148,6 +159,9 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
const canvas = document.createElement('canvas')
const sourceWidth = video.videoWidth
const sourceHeight = video.videoHeight
if (sourceWidth <= 0 || sourceHeight <= 0)
return null
const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1)
canvas.width = Math.round(sourceWidth * scale)
canvas.height = Math.round(sourceHeight * scale)
apps/stage-tamagotchi/src/renderer/pages/devtools/screen-capture.vue
+5 -14
View File
@@ -9,6 +9,8 @@ import { useI18n } from 'vue-i18n'
import WithScreenCapture from '../../components/WithScreenCapture.vue'
import { createObjectUrlFromBytes } from '../../utils/create-object-url-from-bytes'
interface ScreenCaptureSource extends SerializableDesktopCapturerSource {
appIconURL?: string
thumbnailURL?: string
@@ -74,17 +76,6 @@ function getShareLabel(source: ScreenCaptureSource) {
return 'Share Window'
}
function toLocalArrayBuffer(bytes: Uint8Array) {
if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) {
return bytes.slice().buffer
}
return bytes.buffer as ArrayBuffer
}
function toObjectUrl(bytes: Uint8Array, mime: string) {
return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime }))
}
async function startCapture(source: SerializableDesktopCapturerSource) {
try {
await selectWithSource(
@@ -139,8 +130,8 @@ async function refetchSources() {
// NOTICE(@sumimakito): Not only thumbnail is empty, the appIcon could be empty as well with nothing returned.
// REVIEW(@sumimakito): This has nothing to do with our side, probably related to a Electron bug, you can
// read more here https://github.com/electron/electron/issues/44504
appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
appIconURL: source.appIcon && source.appIcon.length > 0 ? createObjectUrlFromBytes(source.appIcon, 'image/png') : undefined,
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? createObjectUrlFromBytes(source.thumbnail, 'image/jpeg') : undefined,
}))
}
catch (err) {
@@ -153,7 +144,7 @@ async function refetchSources() {
}
onMounted(async () => {
refetchSources()
await refetchSources()
})
onBeforeUnmount(() => {
apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue
+19 -17
View File
@@ -14,7 +14,7 @@ import WithScreenCapture from '../../components/WithScreenCapture.vue'
import { useVisionScreenCapture } from '../../composables/use-vision-screen-capture'
type SourceCategory = 'applications' | 'displays' | 'devices'
type SourceCategory = 'applications' | 'displays'
const visionStore = useVisionStore()
const visionProcessingStore = useVisionProcessingStore()
@@ -68,7 +68,6 @@ const {
const categoryOptions = [
{ label: 'Applications', value: 'applications', icon: 'i-solar:window-frame-line-duotone' },
{ label: 'Displays', value: 'displays', icon: 'i-solar:screencast-2-line-duotone' },
{ label: 'Devices', value: 'devices', icon: 'i-solar:smartphone-2-line-duotone' },
]
const workloadOptions = VISION_WORKLOADS.map(workload => ({
@@ -78,27 +77,21 @@ const workloadOptions = VISION_WORKLOADS.map(workload => ({
const isDisplaySource = (source: { id: string }) => source.id.startsWith('screen:')
const isWindowSource = (source: { id: string }) => source.id.startsWith('window:')
const isDeviceSource = (source: { id: string }) => source.id.startsWith('device:')
const filteredSources = computed(() => {
if (sourceCategory.value === 'applications')
return sources.value.filter(isWindowSource)
if (sourceCategory.value === 'displays')
return sources.value.filter(isDisplaySource)
return sources.value.filter(isDeviceSource)
return sources.value.filter(isDisplaySource)
})
const sourceCounts = computed(() => ({
applications: sources.value.filter(isWindowSource).length,
displays: sources.value.filter(isDisplaySource).length,
devices: sources.value.filter(isDeviceSource).length,
}))
function getShareLabel(source: { id: string }) {
if (isDisplaySource(source))
return 'Share Screen'
if (isDeviceSource(source))
return 'Share Device'
return 'Share Window'
}
@@ -154,7 +147,13 @@ async function ensureVideoStream() {
resolve()
return
}
video.onloadedmetadata = () => resolve()
const handleLoadedMetadata = () => {
video.removeEventListener('loadedmetadata', handleLoadedMetadata)
resolve()
}
video.addEventListener('loadedmetadata', handleLoadedMetadata)
})
}
@@ -164,7 +163,7 @@ async function handleVisionTick() {
try {
if (!hasLiveVideoStream(activeStream.value)) {
activeStream.value = null
stopStream()
await ensureVideoStream()
}
@@ -253,6 +252,10 @@ async function shareSource(sourceId: string) {
}
}
function handlePermissionGranted() {
void refetchSources()
}
onBeforeUnmount(() => {
visionProcessingStore.stopTicker()
stopStream()
@@ -261,16 +264,15 @@ onBeforeUnmount(() => {
</script>
<template>
<WithScreenCapture :sources-options="sourcesOptions">
<WithScreenCapture
:sources-options="sourcesOptions"
@permission-granted="handlePermissionGranted()"
>
<template #default="{ hasPermissions, requestPermission }">
<div
v-if="hasPermissions"
:class="['flex', 'flex-col', 'gap-6']"
>
<div v-if="!hasFetchedOnce && !isRefetching" :class="['hidden']">
{{ refetchSources() }}
</div>
<div :class="['flex', 'items-center', 'justify-between', 'rounded-xl', 'bg-neutral-100', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
<div :class="['flex', 'flex-col', 'gap-1']">
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
@@ -549,7 +551,7 @@ onBeforeUnmount(() => {
<div :class="['rounded-xl', 'bg-neutral-100', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
<div :class="['flex', 'items-center', 'justify-between', 'text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
<span>Snapshot</span>
<span>{{ captureCount }} captures · {{ contextUpdateCount }} context updates</span>
<span>{{ captureCount }} captures, {{ contextUpdateCount }} context updates</span>
</div>
<div
v-if="screenshotDataUrl"
apps/stage-tamagotchi/src/renderer/utils/create-object-url-from-bytes.ts
+17
View File
@@ -0,0 +1,17 @@
/**
* Creates a blob URL from a byte view without leaking unrelated backing-buffer bytes.
*
* Use when:
* - Electron screen-capture APIs return `Uint8Array` thumbnails or app icons
* - The byte view may point at a sliced `ArrayBuffer` or `SharedArrayBuffer`
*
* Expects:
* - `bytes` to contain only the payload that should be exposed through the blob URL
*
* Returns:
* - A `blob:` URL that the caller must revoke when it is no longer needed
*/
export function createObjectUrlFromBytes(bytes: Uint8Array, mime: string): string {
const ownedBytes = Uint8Array.from(bytes)
return URL.createObjectURL(new Blob([ownedBytes], { type: mime }))
}