From 2dd2a149b8f5a1d49f9ce03d3b0ee87738fc3818 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Fri, 9 Jan 2026 03:24:48 +0800 Subject: [PATCH] feat(stage-tamagotchi): added fundamental vision --- apps/stage-pocket/package.json | 1 + apps/stage-tamagotchi/package.json | 1 + .../windows/settings/rpc/index.electron.ts | 15 +- .../renderer/components/WithScreenCapture.vue | 2 + .../composables/use-vision-screen-capture.ts | 146 ++++ .../src/renderer/pages/devtools/vision.vue | 501 ++++++++++++++ .../pages/settings/system/developer.vue | 12 + apps/stage-tamagotchi/src/shared/eventa.ts | 2 + apps/stage-web/package.json | 1 + .../src/pages/settings/system/developer.vue | 6 + packages/stage-pages/package.json | 1 + .../src/pages/devtools/context-observer.vue | 645 ++++++++++++++++++ .../src/pages/settings/modules/vision.vue | 383 ++++++++++- .../src/pages/settings/providers/index.vue | 2 +- packages/stage-ui/package.json | 3 + .../stage-ui/src/components/gadgets/index.ts | 2 + .../gadgets/processing-meter.story.vue | 63 ++ .../components/gadgets/processing-meter.vue | 116 ++++ .../gadgets/timeline-range.story.vue | 45 ++ .../src/components/gadgets/timeline-range.vue | 179 +++++ packages/stage-ui/src/composables/index.ts | 1 + .../src/composables/use-modules-list.ts | 4 +- .../stage-ui/src/composables/vision/index.ts | 2 + .../vision/use-vision-inference.ts | 82 +++ .../vision/use-vision-workloads.ts | 55 ++ .../stage-ui/src/stores/chat/context-store.ts | 18 + packages/stage-ui/src/stores/modules/index.ts | 1 + .../src/stores/modules/vision/agents/index.ts | 7 + .../src/stores/modules/vision/index.ts | 5 + .../src/stores/modules/vision/orchestrator.ts | 93 +++ .../stores/modules/vision/processing-store.ts | 204 ++++++ .../src/stores/modules/vision/store.ts | 93 +++ .../modules/vision/utils/model-catalog.ts | 55 ++ pnpm-lock.yaml | 547 +++++++++++++++ pnpm-workspace.yaml | 2 + 35 files changed, 3288 insertions(+), 7 deletions(-) create mode 100644 apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts create mode 100644 apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue create mode 100644 packages/stage-pages/src/pages/devtools/context-observer.vue create mode 100644 packages/stage-ui/src/components/gadgets/processing-meter.story.vue create mode 100644 packages/stage-ui/src/components/gadgets/processing-meter.vue create mode 100644 packages/stage-ui/src/components/gadgets/timeline-range.story.vue create mode 100644 packages/stage-ui/src/components/gadgets/timeline-range.vue create mode 100644 packages/stage-ui/src/composables/vision/index.ts create mode 100644 packages/stage-ui/src/composables/vision/use-vision-inference.ts create mode 100644 packages/stage-ui/src/composables/vision/use-vision-workloads.ts create mode 100644 packages/stage-ui/src/stores/modules/vision/agents/index.ts create mode 100644 packages/stage-ui/src/stores/modules/vision/index.ts create mode 100644 packages/stage-ui/src/stores/modules/vision/orchestrator.ts create mode 100644 packages/stage-ui/src/stores/modules/vision/processing-store.ts create mode 100644 packages/stage-ui/src/stores/modules/vision/store.ts create mode 100644 packages/stage-ui/src/stores/modules/vision/utils/model-catalog.ts diff --git a/apps/stage-pocket/package.json b/apps/stage-pocket/package.json index e54f9df70..773cff92c 100644 --- a/apps/stage-pocket/package.json +++ b/apps/stage-pocket/package.json @@ -63,6 +63,7 @@ "capacitor-native-settings": "catalog:", "colorjs.io": "^0.6.1", "culori": "^4.0.2", + "d3": "catalog:", "date-fns": "^4.1.0", "dompurify": "^3.3.1", "driver.js": "^1.4.0", diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index 29d52093f..ec2a92d63 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -86,6 +86,7 @@ "async-mutex": "catalog:", "colorjs.io": "^0.6.1", "culori": "^4.0.2", + "d3": "catalog:", "date-fns": "^4.1.0", "defu": "^6.1.4", "destr": "^2.0.5", diff --git a/apps/stage-tamagotchi/src/main/windows/settings/rpc/index.electron.ts b/apps/stage-tamagotchi/src/main/windows/settings/rpc/index.electron.ts index b56e4f422..68e6240cb 100644 --- a/apps/stage-tamagotchi/src/main/windows/settings/rpc/index.electron.ts +++ b/apps/stage-tamagotchi/src/main/windows/settings/rpc/index.electron.ts @@ -9,9 +9,9 @@ import type { WidgetsWindowManager } from '../../widgets' import { defineInvokeHandler } from '@moeru/eventa' import { createContext } from '@moeru/eventa/adapters/electron/main' -import { ipcMain } from 'electron' +import { desktopCapturer, ipcMain, session } from 'electron' -import { electronOpenDevtoolsWindow, electronOpenSettingsDevtools } from '../../../../shared/eventa' +import { electronOpenDevtoolsWindow, electronOpenSettingsDevtools, modulesVisionPrepareScreenSourceSelection } from '../../../../shared/eventa' import { createMcpServersService } from '../../../services/airi/mcp-servers' import { createWidgetsService } from '../../../services/airi/widgets' import { createAutoUpdaterService } from '../../../services/electron' @@ -44,5 +44,16 @@ export async function setupSettingsWindowInvokes(params: { await params.devtoolsMarkdownStressWindow.openWindow(payload?.route) }) + defineInvokeHandler(context, modulesVisionPrepareScreenSourceSelection, async () => { + // TODO(@sumimakito): Refactor electron-audio-loopback first then move this to register for beat-sync handler. + // TODO(@nekomeowww): Currently, beat-sync and vision cannot be used together, as they both overriding the display media request handler. + session.defaultSession.setDisplayMediaRequestHandler((_request, callback) => { + desktopCapturer.getSources({ types: ['screen'] }).then((sources) => { + // Grant access to the first screen found. + callback({ video: sources[0], audio: 'loopback' }) + }) + }, { useSystemPicker: false }) + }) + return context } diff --git a/apps/stage-tamagotchi/src/renderer/components/WithScreenCapture.vue b/apps/stage-tamagotchi/src/renderer/components/WithScreenCapture.vue index 5655123f2..2c04b6a7c 100644 --- a/apps/stage-tamagotchi/src/renderer/components/WithScreenCapture.vue +++ b/apps/stage-tamagotchi/src/renderer/components/WithScreenCapture.vue @@ -21,6 +21,7 @@ const { t } = useI18n() const { getSources, setSource, + resetSource, selectWithSource, checkMacOSPermission, requestMacOSPermission, @@ -62,6 +63,7 @@ watch(focused, async (isFocused) => { v-bind="{ getSources, setSource, + resetSource, selectWithSource, hasPermissions, checkPermissions, diff --git a/apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts b/apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts new file mode 100644 index 000000000..5f7c6ec39 --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/composables/use-vision-screen-capture.ts @@ -0,0 +1,146 @@ +import type { SerializableDesktopCapturerSource } from '@proj-airi/electron-screen-capture' +import type { SourcesOptions } from 'electron' +import type { MaybeRefOrGetter } from 'vue' + +import { useElectronScreenCapture } from '@proj-airi/electron-screen-capture/vue' +import { computed, ref, toRaw, toValue } from 'vue' + +interface ScreenCaptureSource extends SerializableDesktopCapturerSource { + appIconURL?: string + thumbnailURL?: string +} + +function toLocalArrayBuffer(bytes: Uint8Array) { + if (typeof SharedArrayBuffer !== 'undefined' && bytes.buffer instanceof SharedArrayBuffer) { + return bytes.slice().buffer + } + return bytes.buffer as ArrayBuffer +} + +function toObjectUrl(bytes: Uint8Array, mime: string) { + return URL.createObjectURL(new Blob([toLocalArrayBuffer(bytes)], { type: mime })) +} + +export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter) { + const sources = ref([]) + const isRefetching = ref(false) + const hasFetchedOnce = ref(false) + const activeSourceId = ref('') + const activeStream = ref(null) + + const { + getSources, + setSource, + resetSource, + } = useElectronScreenCapture(window.electron.ipcRenderer, sourcesOptions) + + const activeSource = computed(() => sources.value.find(source => source.id === activeSourceId.value) || null) + + async function refetchSources() { + try { + isRefetching.value = true + const nextSources = (await getSources()) + .sort((a, b) => { + const aIsScreen = a.id.startsWith('screen:') + const bIsScreen = b.id.startsWith('screen:') + if (aIsScreen !== bIsScreen) + return aIsScreen ? -1 : 1 + return a.name.localeCompare(b.name) + }) + + sources.value.forEach((oldSource) => { + if (oldSource.appIconURL) + URL.revokeObjectURL(oldSource.appIconURL) + if (oldSource.thumbnailURL) + URL.revokeObjectURL(oldSource.thumbnailURL) + }) + + sources.value = nextSources.map(source => ({ + ...source, + appIconURL: source.appIcon && source.appIcon.length > 0 ? toObjectUrl(source.appIcon, 'image/png') : undefined, + thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined, + })) + + if (!activeSourceId.value && sources.value.length > 0) { + activeSourceId.value = sources.value[0]?.id || '' + } + } + finally { + isRefetching.value = false + hasFetchedOnce.value = true + } + } + + async function startStream() { + if (!activeSourceId.value) + throw new Error('No active source selected') + + if (activeStream.value) { + activeStream.value.getTracks().forEach(track => track.stop()) + } + + const handle = await setSource({ + options: toRaw(toValue(sourcesOptions)), + sourceId: activeSourceId.value, + }) + + try { + const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }) + activeStream.value = stream + return stream + } + finally { + await resetSource(handle) + } + } + + function stopStream() { + if (activeStream.value) { + activeStream.value.getTracks().forEach(track => track.stop()) + } + activeStream.value = null + } + + function cleanup() { + stopStream() + sources.value.forEach((oldSource) => { + if (oldSource.appIconURL) + URL.revokeObjectURL(oldSource.appIconURL) + if (oldSource.thumbnailURL) + URL.revokeObjectURL(oldSource.thumbnailURL) + }) + } + + function captureFrame(video: HTMLVideoElement, quality = 0.82, maxWidth = 1280, maxHeight = 720) { + if (!video || video.readyState < 2) + return null + + const canvas = document.createElement('canvas') + const sourceWidth = video.videoWidth + const sourceHeight = video.videoHeight + const scale = Math.min(maxWidth / sourceWidth, maxHeight / sourceHeight, 1) + canvas.width = Math.round(sourceWidth * scale) + canvas.height = Math.round(sourceHeight * scale) + + const ctx = canvas.getContext('2d') + if (!ctx) + throw new Error('Failed to create canvas context') + + ctx.drawImage(video, 0, 0, canvas.width, canvas.height) + return canvas.toDataURL('image/jpeg', quality) + } + + return { + sources, + activeSourceId, + activeSource, + activeStream, + isRefetching, + hasFetchedOnce, + refetchSources, + startStream, + stopStream, + cleanup, + captureFrame, + } +} diff --git a/apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue b/apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue new file mode 100644 index 000000000..f84c031da --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/pages/devtools/vision.vue @@ -0,0 +1,501 @@ + + +