chore(stage-tamagotchi,stage-pages,stage-layouts): vision devtools improved
This commit is contained in:
Neko Ayaka
@@ -36,6 +36,33 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
|
||||
|
||||
const activeSource = computed(() => sources.value.find(source => source.id === activeSourceId.value) || null)
|
||||
|
||||
function isActiveStream(stream: MediaStream | null | undefined) {
|
||||
if (!stream)
|
||||
return false
|
||||
|
||||
return stream.getVideoTracks().some(track => track.readyState === 'live')
|
||||
}
|
||||
|
||||
function clearActiveStream() {
|
||||
const stream = activeStream.value
|
||||
if (!stream) {
|
||||
activeStream.value = null
|
||||
return
|
||||
}
|
||||
|
||||
stream.getTracks().forEach(track => track.stop())
|
||||
activeStream.value = null
|
||||
}
|
||||
|
||||
function attachStreamLifecycle(stream: MediaStream) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
track.addEventListener('ended', () => {
|
||||
if (activeStream.value === stream)
|
||||
activeStream.value = null
|
||||
}, { once: true })
|
||||
})
|
||||
}
|
||||
|
||||
async function refetchSources() {
|
||||
try {
|
||||
isRefetching.value = true
|
||||
@@ -61,9 +88,9 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
|
||||
thumbnailURL: source.thumbnail && source.thumbnail.length > 0 ? toObjectUrl(source.thumbnail, 'image/jpeg') : undefined,
|
||||
}))
|
||||
|
||||
if (!activeSourceId.value && sources.value.length > 0) {
|
||||
const hasActiveSource = sources.value.some(source => source.id === activeSourceId.value)
|
||||
if (!hasActiveSource)
|
||||
activeSourceId.value = sources.value[0]?.id || ''
|
||||
}
|
||||
}
|
||||
finally {
|
||||
isRefetching.value = false
|
||||
@@ -75,9 +102,10 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
|
||||
if (!activeSourceId.value)
|
||||
throw new Error('No active source selected')
|
||||
|
||||
if (activeStream.value) {
|
||||
activeStream.value.getTracks().forEach(track => track.stop())
|
||||
}
|
||||
if (isActiveStream(activeStream.value))
|
||||
return activeStream.value!
|
||||
|
||||
clearActiveStream()
|
||||
|
||||
const handle = await setSource({
|
||||
options: toRaw(toValue(sourcesOptions)),
|
||||
@@ -87,18 +115,20 @@ export function useVisionScreenCapture(sourcesOptions: MaybeRefOrGetter<SourcesO
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false })
|
||||
activeStream.value = stream
|
||||
attachStreamLifecycle(stream)
|
||||
return stream
|
||||
}
|
||||
catch (error) {
|
||||
activeStream.value = null
|
||||
throw error
|
||||
}
|
||||
finally {
|
||||
await resetSource(handle)
|
||||
}
|
||||
}
|
||||
|
||||
function stopStream() {
|
||||
if (activeStream.value) {
|
||||
activeStream.value.getTracks().forEach(track => track.stop())
|
||||
}
|
||||
activeStream.value = null
|
||||
clearActiveStream()
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
|
||||
@@ -130,6 +130,13 @@ const expectedRateMax = computed(() => {
|
||||
return Math.max(60, Math.ceil(60000 / interval))
|
||||
})
|
||||
|
||||
function hasLiveVideoStream(stream: MediaStream | null) {
|
||||
if (!stream)
|
||||
return false
|
||||
|
||||
return stream.getVideoTracks().some(track => track.readyState === 'live')
|
||||
}
|
||||
|
||||
async function ensureVideoStream() {
|
||||
if (!activeSourceId.value)
|
||||
return
|
||||
@@ -156,8 +163,10 @@ async function handleVisionTick() {
|
||||
return
|
||||
|
||||
try {
|
||||
if (!activeStream.value)
|
||||
if (!hasLiveVideoStream(activeStream.value)) {
|
||||
activeStream.value = null
|
||||
await ensureVideoStream()
|
||||
}
|
||||
|
||||
const video = videoRef.value
|
||||
if (!video)
|
||||
|
||||
@@ -43,12 +43,6 @@ const menu = computed(() => [
|
||||
icon: 'i-solar:chat-square-call-bold-duotone',
|
||||
to: '/devtools/context-flow',
|
||||
},
|
||||
{
|
||||
title: 'Context Observer',
|
||||
description: 'Track how active context entries merge over time',
|
||||
icon: 'i-solar:chat-square-bold-duotone',
|
||||
to: '/devtools/context-observer',
|
||||
},
|
||||
{
|
||||
title: 'Relative Mouse',
|
||||
description: 'Get mouse position relative to the window',
|
||||
@@ -100,42 +94,27 @@ const openDevtoolsWindow = useElectronEventaInvoke(electronOpenDevtoolsWindow)
|
||||
<template>
|
||||
<ButtonBar
|
||||
v-model="settings.disableTransitions"
|
||||
v-motion
|
||||
mb-2
|
||||
icon="i-solar:settings-minimalistic-outline"
|
||||
text="settings.pages.page.developers.open-devtools.title"
|
||||
:initial="{ opacity: 0, y: 10 }"
|
||||
:enter="{ opacity: 1, y: 0 }"
|
||||
:duration="250 + (19 * 10)"
|
||||
:delay="1 * 50"
|
||||
transition="all ease-in-out duration-250"
|
||||
@click="() => openDevTools()"
|
||||
>
|
||||
{{ t('settings.pages.page.developers.open-devtools.button') }}
|
||||
</ButtonBar>
|
||||
<ButtonBar
|
||||
v-motion
|
||||
mb-2
|
||||
icon="i-solar:code-bold-duotone"
|
||||
:text="t('tamagotchi.settings.devtools.pages.markdown-stress.title')"
|
||||
:initial="{ opacity: 0, y: 10 }"
|
||||
:enter="{ opacity: 1, y: 0 }"
|
||||
:duration="250 + (19 * 10)"
|
||||
:delay="2 * 50"
|
||||
transition="all ease-in-out duration-250"
|
||||
@click="() => openDevtoolsWindow({ route: '/devtools/markdown-stress' })"
|
||||
>
|
||||
{{ t('tamagotchi.settings.devtools.pages.markdown-stress.title') }}
|
||||
</ButtonBar>
|
||||
<ButtonBar
|
||||
v-motion
|
||||
mb-2
|
||||
icon="i-solar:chart-square-bold-duotone"
|
||||
:text="t('tamagotchi.settings.devtools.pages.lag-visualizer.title')"
|
||||
:initial="{ opacity: 0, y: 10 }"
|
||||
:enter="{ opacity: 1, y: 0 }"
|
||||
:duration="250 + (19 * 10)"
|
||||
:delay="3 * 50"
|
||||
transition="all ease-in-out duration-250"
|
||||
@click="() => router.push('/devtools/performance-visualizer')"
|
||||
>
|
||||
@@ -143,29 +122,19 @@ const openDevtoolsWindow = useElectronEventaInvoke(electronOpenDevtoolsWindow)
|
||||
</ButtonBar>
|
||||
<CheckBar
|
||||
v-model="settings.disableTransitions"
|
||||
v-motion
|
||||
mb-2
|
||||
icon-on="i-solar:people-nearby-bold-duotone"
|
||||
icon-off="i-solar:running-2-line-duotone"
|
||||
text="settings.animations.stage-transitions.title"
|
||||
:initial="{ opacity: 0, y: 10 }"
|
||||
:enter="{ opacity: 1, y: 0 }"
|
||||
:duration="250 + (19 * 10)"
|
||||
:delay="4 * 50"
|
||||
transition="all ease-in-out duration-250"
|
||||
/>
|
||||
<CheckBar
|
||||
v-model="settings.usePageSpecificTransitions"
|
||||
v-motion
|
||||
:disabled="settings.disableTransitions"
|
||||
icon-on="i-solar:running-2-line-duotone"
|
||||
icon-off="i-solar:people-nearby-bold-duotone"
|
||||
text="settings.animations.use-page-specific-transitions.title"
|
||||
description="settings.animations.use-page-specific-transitions.description"
|
||||
:initial="{ opacity: 0, y: 10 }"
|
||||
:enter="{ opacity: 1, y: 0 }"
|
||||
:duration="250 + (20 * 10)"
|
||||
:delay="5 * 50"
|
||||
transition="all ease-in-out duration-250"
|
||||
/>
|
||||
|
||||
@@ -178,7 +147,7 @@ const openDevtoolsWindow = useElectronEventaInvoke(electronOpenDevtoolsWindow)
|
||||
:enter="{ opacity: 1, y: 0 }"
|
||||
:duration="250"
|
||||
:style="{
|
||||
transitionDelay: `${(5 + index) * 50}ms`, // delay between each item, unocss doesn't support dynamic generation of classes now
|
||||
transitionDelay: `${(index) * 50}ms`, // delay between each item, unocss doesn't support dynamic generation of classes now
|
||||
}"
|
||||
:title="item.title"
|
||||
:description="item.description"
|
||||
|
||||
@@ -68,12 +68,6 @@ const menu = computed(() => [
|
||||
icon: 'i-solar:chat-square-call-bold-duotone',
|
||||
to: '/devtools/context-flow',
|
||||
},
|
||||
{
|
||||
title: 'Context Observer',
|
||||
description: 'Track how active context entries merge over time',
|
||||
icon: 'i-solar:chat-square-bold-duotone',
|
||||
to: '/devtools/context-observer',
|
||||
},
|
||||
{
|
||||
title: 'WebSocket Inspector',
|
||||
description: 'Inspect raw WebSocket traffic',
|
||||
|
||||
@@ -337,9 +337,6 @@ pages:
|
||||
subtitle: Select a default model from the provider
|
||||
title: Model
|
||||
title: Consciousness
|
||||
context-observer:
|
||||
description: Track how active context entries merge over time
|
||||
title: Context Observer
|
||||
description: Thinking, vision, speech synthesis, gaming, etc.
|
||||
gaming-factorio:
|
||||
description: Playing Factorio!
|
||||
|
||||
@@ -1,652 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
|
||||
import { TimelineRange, TimeSeriesChart } from '@proj-airi/stage-ui/components'
|
||||
import { useChatContextStore } from '@proj-airi/stage-ui/stores/chat/context-store'
|
||||
import { Button, FieldCheckbox, FieldInput, FieldRange, FieldSelect } from '@proj-airi/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { TooltipArrow, TooltipContent, TooltipPortal, TooltipProvider, TooltipRoot, TooltipTrigger } from 'reka-ui'
|
||||
import { computed, ref, watch } from 'vue'
|
||||
|
||||
const chatContextStore = useChatContextStore()
|
||||
const { activeContexts, contextHistory } = storeToRefs(chatContextStore)
|
||||
|
||||
const selectedSource = ref('all')
|
||||
const strategyFilter = ref<'all' | ContextUpdateStrategy>('all')
|
||||
const textFilter = ref('')
|
||||
const onlyWithText = ref(false)
|
||||
const maxHistoryEntries = ref(220)
|
||||
const followLatest = ref(true)
|
||||
const showMetadata = ref(false)
|
||||
const rangeSelection = ref<[number, number] | null>(null)
|
||||
|
||||
const selectedEntryId = ref<string | null>(null)
|
||||
|
||||
const sourceOptions = computed(() => {
|
||||
const entries = Object.keys(activeContexts.value).sort()
|
||||
return [
|
||||
{ label: 'All modules', value: 'all' },
|
||||
...entries.map(source => ({ label: source, value: source })),
|
||||
]
|
||||
})
|
||||
|
||||
const strategyOptions = [
|
||||
{ label: 'All strategies', value: 'all' },
|
||||
{ label: 'Replace', value: ContextUpdateStrategy.ReplaceSelf },
|
||||
{ label: 'Append', value: ContextUpdateStrategy.AppendSelf },
|
||||
]
|
||||
|
||||
const baseHistory = computed(() => {
|
||||
const query = textFilter.value.trim().toLowerCase()
|
||||
|
||||
return contextHistory.value
|
||||
.filter((entry) => {
|
||||
if (selectedSource.value === 'all')
|
||||
return true
|
||||
return entry.sourceKey === selectedSource.value
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (strategyFilter.value === 'all')
|
||||
return true
|
||||
return entry.strategy === strategyFilter.value
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (!onlyWithText.value)
|
||||
return true
|
||||
return Boolean(entry.text?.trim())
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (!query)
|
||||
return true
|
||||
return [
|
||||
entry.sourceKey,
|
||||
entry.text,
|
||||
entry.id,
|
||||
entry.contextId,
|
||||
].filter(Boolean).join(' ').toLowerCase().includes(query)
|
||||
})
|
||||
.slice(-maxHistoryEntries.value)
|
||||
})
|
||||
|
||||
const filteredHistory = computed(() => {
|
||||
if (!rangeSelection.value)
|
||||
return baseHistory.value
|
||||
|
||||
const [start, end] = rangeSelection.value
|
||||
return baseHistory.value.filter((entry) => {
|
||||
const timestamp = entry.createdAt ?? 0
|
||||
return timestamp >= start && timestamp <= end
|
||||
})
|
||||
})
|
||||
|
||||
const rangeData = computed(() => baseHistory.value.map(entry => entry.createdAt ?? 0))
|
||||
|
||||
const activeContextGroups = computed(() => {
|
||||
const keys = selectedSource.value === 'all'
|
||||
? Object.keys(activeContexts.value)
|
||||
: [selectedSource.value]
|
||||
|
||||
return keys
|
||||
.sort()
|
||||
.map((key) => {
|
||||
const contexts = activeContexts.value[key] ?? []
|
||||
const latest = contexts.at(-1)?.createdAt ?? 0
|
||||
return {
|
||||
key,
|
||||
contexts,
|
||||
latest,
|
||||
}
|
||||
})
|
||||
.sort((a, b) => b.latest - a.latest)
|
||||
})
|
||||
|
||||
const totalActiveContexts = computed(() => {
|
||||
return Object.values(activeContexts.value).reduce((sum, contexts) => sum + contexts.length, 0)
|
||||
})
|
||||
|
||||
const updateWindowMs = 60_000
|
||||
const updateBucketCount = 30
|
||||
const updateBucketMs = updateWindowMs / updateBucketCount
|
||||
|
||||
const updateTimeline = computed(() => {
|
||||
const now = Date.now()
|
||||
const buckets = Array.from({ length: updateBucketCount }, () => 0)
|
||||
|
||||
for (const entry of filteredHistory.value) {
|
||||
const ts = entry.createdAt || now
|
||||
const offset = ts - (now - updateWindowMs)
|
||||
if (offset < 0 || offset > updateWindowMs)
|
||||
continue
|
||||
const index = Math.min(updateBucketCount - 1, Math.max(0, Math.floor(offset / updateBucketMs)))
|
||||
buckets[index] += 1
|
||||
}
|
||||
|
||||
const peak = Math.max(1, ...buckets)
|
||||
const normalized = buckets.map(value => value / peak)
|
||||
return {
|
||||
normalized,
|
||||
currentValue: normalized[normalized.length - 1] ?? 0,
|
||||
total: buckets.reduce((sum, value) => sum + value, 0),
|
||||
}
|
||||
})
|
||||
|
||||
const queueEntries = computed(() => filteredHistory.value.slice(-80))
|
||||
|
||||
const selectedEntry = computed(() => {
|
||||
if (!selectedEntryId.value)
|
||||
return null
|
||||
return filteredHistory.value.find(entry => entry.id === selectedEntryId.value) || null
|
||||
})
|
||||
|
||||
const activeFilterSummary = computed(() => {
|
||||
const filters = []
|
||||
if (selectedSource.value !== 'all')
|
||||
filters.push(`module:${selectedSource.value}`)
|
||||
if (strategyFilter.value !== 'all')
|
||||
filters.push(`strategy:${formatStrategy(strategyFilter.value)}`)
|
||||
if (onlyWithText.value)
|
||||
filters.push('text-only')
|
||||
if (textFilter.value.trim())
|
||||
filters.push(`query:${textFilter.value.trim()}`)
|
||||
if (rangeSelection.value) {
|
||||
const [start, end] = rangeSelection.value
|
||||
filters.push(`${new Date(start).toLocaleTimeString()} → ${new Date(end).toLocaleTimeString()}`)
|
||||
}
|
||||
return filters.length ? filters.join(' · ') : 'All updates'
|
||||
})
|
||||
|
||||
function formatStrategy(strategy: ContextUpdateStrategy) {
|
||||
return strategy === ContextUpdateStrategy.AppendSelf ? 'Append' : 'Replace'
|
||||
}
|
||||
|
||||
function formatTimestamp(timestamp: number) {
|
||||
return new Date(timestamp).toLocaleTimeString()
|
||||
}
|
||||
|
||||
function formatRelativeTime(timestamp: number | null | undefined) {
|
||||
if (!timestamp)
|
||||
return 'Never'
|
||||
const diffMs = Date.now() - timestamp
|
||||
const diffSeconds = Math.max(0, Math.floor(diffMs / 1000))
|
||||
if (diffSeconds < 60)
|
||||
return `${diffSeconds}s ago`
|
||||
const diffMinutes = Math.floor(diffSeconds / 60)
|
||||
if (diffMinutes < 60)
|
||||
return `${diffMinutes}m ago`
|
||||
const diffHours = Math.floor(diffMinutes / 60)
|
||||
return `${diffHours}h ago`
|
||||
}
|
||||
|
||||
function summarizeText(text: string, limit = 120) {
|
||||
if (!text)
|
||||
return 'No context text.'
|
||||
if (text.length <= limit)
|
||||
return text
|
||||
return `${text.slice(0, limit)}…`
|
||||
}
|
||||
|
||||
function queueDotClass(entry: { strategy: ContextUpdateStrategy }) {
|
||||
if (entry.strategy === ContextUpdateStrategy.AppendSelf)
|
||||
return 'bg-emerald-400/80'
|
||||
return 'bg-amber-400/80'
|
||||
}
|
||||
|
||||
function resetRange() {
|
||||
rangeSelection.value = null
|
||||
}
|
||||
|
||||
watch(() => filteredHistory.value.length, () => {
|
||||
if (!followLatest.value)
|
||||
return
|
||||
const latest = filteredHistory.value.at(-1)
|
||||
if (latest)
|
||||
selectedEntryId.value = latest.id
|
||||
})
|
||||
|
||||
watch([selectedSource, strategyFilter, textFilter, onlyWithText], () => {
|
||||
if (!followLatest.value)
|
||||
return
|
||||
const latest = filteredHistory.value.at(-1)
|
||||
if (latest)
|
||||
selectedEntryId.value = latest.id
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-gradient-to-br',
|
||||
'from-neutral-100',
|
||||
'via-white',
|
||||
'to-primary-100/50',
|
||||
'p-5',
|
||||
'shadow-sm',
|
||||
'dark:from-neutral-900/80',
|
||||
'dark:via-neutral-950',
|
||||
'dark:to-primary-900/20',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'flex-col', 'gap-5', 'lg:flex-row', 'lg:items-center', 'lg:justify-between']">
|
||||
<div :class="['flex', 'flex-col', 'gap-2']">
|
||||
<div :class="['text-xs', 'uppercase', 'tracking-[0.25em]', 'text-neutral-400']">
|
||||
Context Observatory
|
||||
</div>
|
||||
<div :class="['text-2xl', 'font-semibold', 'text-neutral-800', 'dark:text-neutral-100']">
|
||||
Active contexts: {{ totalActiveContexts }}
|
||||
</div>
|
||||
<div :class="['text-sm', 'text-neutral-500', 'dark:text-neutral-400']">
|
||||
{{ activeFilterSummary }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['grid', 'gap-3', 'md:grid-cols-2', 'xl:grid-cols-4']">
|
||||
<FieldSelect
|
||||
v-model="selectedSource"
|
||||
label="Module"
|
||||
:options="sourceOptions"
|
||||
/>
|
||||
<FieldSelect
|
||||
v-model="strategyFilter"
|
||||
label="Strategy"
|
||||
:options="strategyOptions"
|
||||
/>
|
||||
<FieldInput
|
||||
v-model="textFilter"
|
||||
label="Search"
|
||||
placeholder="Find by text / id"
|
||||
/>
|
||||
<FieldRange
|
||||
v-model="maxHistoryEntries"
|
||||
label="Log depth"
|
||||
:min="40"
|
||||
:max="300"
|
||||
:step="20"
|
||||
:format-value="value => `${value} entries`"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-4', 'grid', 'gap-4', 'md:grid-cols-3']">
|
||||
<FieldCheckbox
|
||||
v-model="onlyWithText"
|
||||
label="Text-only updates"
|
||||
description="Hide empty or metadata-only updates."
|
||||
/>
|
||||
<FieldCheckbox
|
||||
v-model="followLatest"
|
||||
label="Follow latest"
|
||||
description="Auto-select newest update."
|
||||
/>
|
||||
<FieldCheckbox
|
||||
v-model="showMetadata"
|
||||
label="Show metadata"
|
||||
description="Display raw metadata in details."
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['grid', 'gap-6', 'xl:grid-cols-[minmax(0,1.1fr)_minmax(0,0.9fr)]']">
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Cadence
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ updateTimeline.total }} updates / min
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<TimeSeriesChart
|
||||
:is-active="false"
|
||||
:history="updateTimeline.normalized"
|
||||
:current-value="updateTimeline.currentValue"
|
||||
:show-header="false"
|
||||
:show-legend="false"
|
||||
:show-current-value="false"
|
||||
:show-area="true"
|
||||
:show-active-indicator="false"
|
||||
:height="96"
|
||||
unit="/min"
|
||||
:precision="0"
|
||||
/>
|
||||
|
||||
<div :class="['mt-4', 'flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Timeline range
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="secondary"
|
||||
:disabled="!rangeSelection"
|
||||
:label="rangeSelection ? 'Reset range' : 'Full range'"
|
||||
@click="resetRange"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<TimelineRange
|
||||
v-model:range="rangeSelection"
|
||||
:data="rangeData"
|
||||
:height="140"
|
||||
:bins="48"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Active contexts
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ activeContextGroups.length }} modules
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-3', 'grid', 'gap-3', 'lg:grid-cols-2']">
|
||||
<div
|
||||
v-for="group in activeContextGroups"
|
||||
:key="group.key"
|
||||
:class="[
|
||||
'rounded-xl',
|
||||
'border',
|
||||
'border-neutral-200',
|
||||
'bg-white',
|
||||
'p-3',
|
||||
'dark:border-neutral-800',
|
||||
'dark:bg-neutral-900',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between', 'gap-2']">
|
||||
<div :class="['text-sm', 'font-semibold', 'text-neutral-700', 'dark:text-neutral-200']">
|
||||
{{ group.key }}
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ group.contexts.length }} contexts · {{ formatRelativeTime(group.latest) }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="group.contexts.length === 0" :class="['mt-2', 'text-xs', 'text-neutral-400']">
|
||||
No active contexts.
|
||||
</div>
|
||||
|
||||
<div v-else :class="['mt-3', 'max-h-60', 'space-y-3', 'overflow-y-auto', 'pr-1']">
|
||||
<div
|
||||
v-for="context in group.contexts.slice().reverse()"
|
||||
:key="context.id"
|
||||
:class="[
|
||||
'rounded-lg',
|
||||
'border',
|
||||
'border-neutral-100',
|
||||
'bg-neutral-50',
|
||||
'p-3',
|
||||
'text-sm',
|
||||
'text-neutral-700',
|
||||
'dark:border-neutral-800/60',
|
||||
'dark:bg-neutral-950/60',
|
||||
'dark:text-neutral-200',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between', 'text-xs', 'text-neutral-400']">
|
||||
<span>{{ formatStrategy(context.strategy) }}</span>
|
||||
<span>{{ context.createdAt ? formatTimestamp(context.createdAt) : 'Unknown' }}</span>
|
||||
</div>
|
||||
<div :class="['mt-2', 'whitespace-pre-wrap', 'text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||
{{ context.text || 'No context text.' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-if="activeContextGroups.length === 0"
|
||||
:class="[
|
||||
'rounded-xl',
|
||||
'border-2',
|
||||
'border-dashed',
|
||||
'border-neutral-200',
|
||||
'px-4',
|
||||
'py-10',
|
||||
'text-sm',
|
||||
'text-neutral-400',
|
||||
'dark:border-neutral-800',
|
||||
]"
|
||||
>
|
||||
No active context modules yet.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Selected update
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ selectedEntry ? formatTimestamp(selectedEntry.createdAt || Date.now()) : 'None' }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="selectedEntry" :class="['mt-3', 'space-y-3']">
|
||||
<div :class="['flex', 'items-center', 'gap-2', 'text-xs', 'text-neutral-400']">
|
||||
<span>{{ selectedEntry.sourceKey }}</span>
|
||||
<span>·</span>
|
||||
<span>{{ formatStrategy(selectedEntry.strategy) }}</span>
|
||||
<span>·</span>
|
||||
<span>{{ selectedEntry.contextId }}</span>
|
||||
</div>
|
||||
<div :class="['whitespace-pre-wrap', 'rounded-lg', 'bg-white', 'p-3', 'text-sm', 'text-neutral-700', 'dark:bg-neutral-900', 'dark:text-neutral-200']">
|
||||
{{ selectedEntry.text || 'No context text.' }}
|
||||
</div>
|
||||
|
||||
<pre
|
||||
v-if="showMetadata"
|
||||
:class="[
|
||||
'rounded-lg',
|
||||
'bg-neutral-900',
|
||||
'p-3',
|
||||
'text-xs',
|
||||
'text-neutral-100',
|
||||
'overflow-auto',
|
||||
]"
|
||||
>
|
||||
{{ JSON.stringify(selectedEntry.metadata ?? {}, null, 2) }}
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
<div v-else :class="['mt-3', 'text-sm', 'text-neutral-400']">
|
||||
Pick a queue dot or stream block to inspect.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Stream blocks
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ queueEntries.length }} recent updates
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-4', 'grid', 'grid-cols-6', 'gap-2', 'sm:grid-cols-8']">
|
||||
<TooltipProvider v-for="entry in queueEntries" :key="entry.id" :delay-duration="120">
|
||||
<TooltipRoot>
|
||||
<TooltipTrigger as-child>
|
||||
<button
|
||||
type="button"
|
||||
:class="[
|
||||
'h-9',
|
||||
'w-9',
|
||||
'rounded-lg',
|
||||
'border',
|
||||
'border-neutral-200',
|
||||
'bg-white',
|
||||
'transition',
|
||||
'duration-150',
|
||||
'dark:border-neutral-800',
|
||||
'dark:bg-neutral-900',
|
||||
selectedEntryId === entry.id ? 'ring-2 ring-primary-500/70' : 'hover:border-neutral-300',
|
||||
]"
|
||||
@click="selectedEntryId = entry.id"
|
||||
>
|
||||
<span :class="['block', 'h-full', 'w-full', 'rounded-lg', queueDotClass(entry)]" />
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipPortal>
|
||||
<TooltipContent
|
||||
:class="[
|
||||
'rounded-md',
|
||||
'bg-white',
|
||||
'px-2',
|
||||
'py-1',
|
||||
'text-xs',
|
||||
'text-neutral-700',
|
||||
'shadow-md',
|
||||
'dark:bg-neutral-900',
|
||||
'dark:text-neutral-200',
|
||||
]"
|
||||
>
|
||||
<div>{{ entry.sourceKey }} · {{ formatStrategy(entry.strategy) }}</div>
|
||||
<div>{{ entry.createdAt ? formatTimestamp(entry.createdAt) : 'Unknown' }}</div>
|
||||
<div>{{ summarizeText(entry.text, 80) }}</div>
|
||||
<TooltipArrow :class="['fill-white', 'dark:fill-neutral-900']" />
|
||||
</TooltipContent>
|
||||
</TooltipPortal>
|
||||
</TooltipRoot>
|
||||
</TooltipProvider>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-4', 'flex', 'flex-col', 'gap-2']">
|
||||
<button
|
||||
v-for="entry in filteredHistory.slice().reverse().slice(0, 12)"
|
||||
:key="entry.id"
|
||||
type="button"
|
||||
:class="[
|
||||
'flex',
|
||||
'items-start',
|
||||
'justify-between',
|
||||
'gap-3',
|
||||
'rounded-xl',
|
||||
'border',
|
||||
'border-neutral-200',
|
||||
'bg-white',
|
||||
'p-3',
|
||||
'text-left',
|
||||
'transition',
|
||||
'duration-150',
|
||||
selectedEntryId === entry.id ? 'border-primary-400/70 shadow-sm' : 'hover:border-neutral-300',
|
||||
'dark:border-neutral-800',
|
||||
'dark:bg-neutral-900',
|
||||
'dark:hover:border-neutral-700',
|
||||
]"
|
||||
@click="selectedEntryId = entry.id"
|
||||
>
|
||||
<div :class="['flex', 'flex-col', 'gap-1']">
|
||||
<div :class="['flex', 'items-center', 'gap-2']">
|
||||
<span :class="['h-2.5', 'w-2.5', 'rounded-full', queueDotClass(entry)]" />
|
||||
<span :class="['text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
{{ entry.sourceKey }}
|
||||
</span>
|
||||
<span :class="['text-xs', 'text-neutral-400']">
|
||||
{{ formatStrategy(entry.strategy) }}
|
||||
</span>
|
||||
</div>
|
||||
<div :class="['text-sm', 'text-neutral-700', 'dark:text-neutral-200']">
|
||||
{{ summarizeText(entry.text, 120) }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<TooltipProvider :delay-duration="150">
|
||||
<TooltipRoot>
|
||||
<TooltipTrigger as-child>
|
||||
<div :class="['text-xs', 'text-neutral-400', 'cursor-help']">
|
||||
{{ entry.createdAt ? formatTimestamp(entry.createdAt) : 'Unknown' }}
|
||||
</div>
|
||||
</TooltipTrigger>
|
||||
<TooltipPortal>
|
||||
<TooltipContent
|
||||
:class="[
|
||||
'rounded-md',
|
||||
'bg-white',
|
||||
'px-2',
|
||||
'py-1',
|
||||
'text-xs',
|
||||
'text-neutral-700',
|
||||
'shadow-md',
|
||||
'dark:bg-neutral-900',
|
||||
'dark:text-neutral-200',
|
||||
]"
|
||||
>
|
||||
{{ entry.createdAt ? new Date(entry.createdAt).toLocaleString() : 'Unknown timestamp' }}
|
||||
<TooltipArrow :class="['fill-white', 'dark:fill-neutral-900']" />
|
||||
</TooltipContent>
|
||||
</TooltipPortal>
|
||||
</TooltipRoot>
|
||||
</TooltipProvider>
|
||||
</button>
|
||||
|
||||
<div
|
||||
v-if="filteredHistory.length === 0"
|
||||
:class="[
|
||||
'rounded-xl',
|
||||
'border-2',
|
||||
'border-dashed',
|
||||
'border-neutral-200',
|
||||
'px-4',
|
||||
'py-10',
|
||||
'text-center',
|
||||
'text-sm',
|
||||
'text-neutral-400',
|
||||
'dark:border-neutral-800',
|
||||
]"
|
||||
>
|
||||
No updates yet. Emit a context update to populate the stream.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<route lang="yaml">
|
||||
meta:
|
||||
layout: settings
|
||||
titleKey: settings.pages.modules.context-observer.title
|
||||
subtitleKey: settings.pages.modules.context-observer.description
|
||||
</route>
|
||||
@@ -5,4 +5,3 @@ export { default as ProcessingMeter } from './processing-meter.vue'
|
||||
export { default as TestDummyMarker } from './test-dummy-marker.vue'
|
||||
export { default as ThresholdMeter } from './threshold-meter.vue'
|
||||
export { default as TimeSeriesChart } from './time-series-chart.vue'
|
||||
export { default as TimelineRange } from './timeline-range.vue'
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import TimelineRange from './timeline-range.vue'
|
||||
|
||||
const data = ref<number[]>([])
|
||||
const range = ref<[number, number] | null>(null)
|
||||
|
||||
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
function seedData() {
|
||||
const now = Date.now()
|
||||
data.value = Array.from({ length: 120 }, (_, index) => now - (120 - index) * 1000)
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
seedData()
|
||||
intervalId = setInterval(() => {
|
||||
data.value = [...data.value.slice(-200), Date.now()]
|
||||
}, 1000)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
if (intervalId)
|
||||
clearInterval(intervalId)
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Story title="Timeline Range" group="gadgets">
|
||||
<Variant id="basic" title="Basic">
|
||||
<div :class="['p-4', 'max-w-3xl']">
|
||||
<TimelineRange
|
||||
v-model:range="range"
|
||||
:data="data"
|
||||
:height="140"
|
||||
:bins="40"
|
||||
/>
|
||||
<div :class="['mt-3', 'text-xs', 'text-neutral-400']">
|
||||
{{ range ? `Range: ${new Date(range[0]).toLocaleTimeString()} - ${new Date(range[1]).toLocaleTimeString()}` : 'No range selected' }}
|
||||
</div>
|
||||
</div>
|
||||
</Variant>
|
||||
</Story>
|
||||
</template>
|
||||
@@ -1,179 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { useElementBounding } from '@vueuse/core'
|
||||
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue'
|
||||
|
||||
import * as d3 from 'd3'
|
||||
|
||||
interface Props {
|
||||
data: number[]
|
||||
height?: number
|
||||
bins?: number
|
||||
range?: [number, number] | null
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
height: 120,
|
||||
bins: 40,
|
||||
range: null,
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
(event: 'update:range', value: [number, number] | null): void
|
||||
}>()
|
||||
|
||||
const containerRef = ref<HTMLDivElement | null>(null)
|
||||
const { width } = useElementBounding(containerRef, { windowResize: true })
|
||||
|
||||
let svg: d3.Selection<SVGSVGElement, unknown, null, undefined> | null = null
|
||||
let brushBehavior: d3.BrushBehavior<unknown> | null = null
|
||||
let xScale: d3.ScaleLinear<number, number> | null = null
|
||||
|
||||
const sanitizedData = computed(() => props.data.filter(value => Number.isFinite(value)))
|
||||
|
||||
function clearChart() {
|
||||
if (!containerRef.value)
|
||||
return
|
||||
containerRef.value.innerHTML = ''
|
||||
svg = null
|
||||
brushBehavior = null
|
||||
xScale = null
|
||||
}
|
||||
|
||||
function drawChart() {
|
||||
if (!containerRef.value)
|
||||
return
|
||||
|
||||
const containerWidth = Math.max(0, Math.floor(width.value || 0))
|
||||
if (containerWidth === 0)
|
||||
return
|
||||
|
||||
clearChart()
|
||||
|
||||
const margin = {
|
||||
top: 12,
|
||||
right: 14,
|
||||
bottom: 20,
|
||||
left: 28,
|
||||
}
|
||||
const chartWidth = Math.max(0, containerWidth - margin.left - margin.right)
|
||||
const chartHeight = Math.max(0, props.height - margin.top - margin.bottom)
|
||||
|
||||
const data = sanitizedData.value
|
||||
if (data.length === 0) {
|
||||
svg = d3.select(containerRef.value)
|
||||
.append('svg')
|
||||
.attr('width', containerWidth)
|
||||
.attr('height', props.height)
|
||||
return
|
||||
}
|
||||
|
||||
const [minValueRaw, maxValueRaw] = d3.extent(data) as [number, number]
|
||||
const minValue = Number.isFinite(minValueRaw) ? minValueRaw : 0
|
||||
const maxValue = Number.isFinite(maxValueRaw) ? maxValueRaw : minValue + 1
|
||||
const paddedMax = maxValue === minValue ? maxValue + 1 : maxValue
|
||||
|
||||
xScale = d3.scaleLinear()
|
||||
.domain([minValue, paddedMax])
|
||||
.range([0, chartWidth])
|
||||
|
||||
const bins = d3.bin()
|
||||
.domain(xScale.domain() as [number, number])
|
||||
.thresholds(props.bins)(data)
|
||||
|
||||
const yScale = d3.scaleLinear()
|
||||
.domain([0, d3.max(bins, bin => bin.length) || 1])
|
||||
.range([chartHeight, 0])
|
||||
|
||||
svg = d3.select(containerRef.value)
|
||||
.append('svg')
|
||||
.attr('width', containerWidth)
|
||||
.attr('height', props.height)
|
||||
.style('overflow', 'visible')
|
||||
|
||||
const chart = svg
|
||||
.append('g')
|
||||
.attr('transform', `translate(${margin.left},${margin.top})`)
|
||||
|
||||
chart
|
||||
.append('g')
|
||||
.attr('class', 'timeline-range-bars')
|
||||
.selectAll('rect')
|
||||
.data(bins)
|
||||
.join('rect')
|
||||
.attr('x', bin => xScale?.(bin.x0 ?? minValue) ?? 0)
|
||||
.attr('y', bin => yScale(bin.length))
|
||||
.attr('width', bin => Math.max(1, (xScale?.(bin.x1 ?? minValue) ?? 0) - (xScale?.(bin.x0 ?? minValue) ?? 0) - 1))
|
||||
.attr('height', bin => chartHeight - yScale(bin.length))
|
||||
.attr('rx', 2)
|
||||
.attr('fill', 'rgba(66,72,83,0.85)')
|
||||
|
||||
const axis = d3.axisBottom(xScale).ticks(4).tickFormat((d) => {
|
||||
if (typeof d === 'number')
|
||||
return new Date(d).toLocaleTimeString()
|
||||
return `${d}`
|
||||
})
|
||||
|
||||
chart
|
||||
.append('g')
|
||||
.attr('class', 'timeline-range-axis')
|
||||
.attr('transform', `translate(0,${chartHeight})`)
|
||||
.call(axis)
|
||||
.call(g => g.select('.domain').attr('opacity', 0.2))
|
||||
.call(g => g.selectAll('text').attr('fill', '#9CA1AE'))
|
||||
.call(g => g.selectAll('line').attr('opacity', 0.08))
|
||||
|
||||
brushBehavior = d3.brushX()
|
||||
.extent([[0, 0], [chartWidth, chartHeight]])
|
||||
.on('brush end', (event) => {
|
||||
if (!xScale)
|
||||
return
|
||||
if (!event.selection) {
|
||||
emit('update:range', null)
|
||||
return
|
||||
}
|
||||
const [start, end] = event.selection as [number, number]
|
||||
emit('update:range', [xScale.invert(start), xScale.invert(end)])
|
||||
})
|
||||
|
||||
const brushGroup = chart
|
||||
.append('g')
|
||||
.attr('class', 'timeline-range-brush')
|
||||
.call(brushBehavior)
|
||||
|
||||
brushGroup.selectAll('.selection')
|
||||
.attr('fill', 'rgba(114,163,183,0.25)')
|
||||
.attr('stroke', 'rgba(114,163,183,0.6)')
|
||||
|
||||
if (props.range) {
|
||||
const [start, end] = props.range
|
||||
brushGroup.call(brushBehavior.move, [xScale(start), xScale(end)])
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
drawChart()
|
||||
})
|
||||
|
||||
watch([width, sanitizedData, () => props.bins, () => props.height], () => {
|
||||
drawChart()
|
||||
})
|
||||
|
||||
watch(() => props.range, (nextRange) => {
|
||||
if (!svg || !brushBehavior || !xScale)
|
||||
return
|
||||
const brushGroup = svg.select<SVGGElement>('g.timeline-range-brush')
|
||||
if (!nextRange) {
|
||||
brushGroup.call(brushBehavior.move, null)
|
||||
return
|
||||
}
|
||||
brushGroup.call(brushBehavior.move, [xScale(nextRange[0]), xScale(nextRange[1])])
|
||||
})
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
clearChart()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div ref="containerRef" :class="['w-full']" />
|
||||
</template>
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { ContextUpdate, WebSocketBaseEvent, WebSocketEvent, WebSocketEventOptionalSource, WebSocketEvents } from '@proj-airi/server-sdk'
|
||||
import type { ContextUpdate, InputContextUpdate, WebSocketBaseEvent, WebSocketEvent, WebSocketEventOptionalSource, WebSocketEvents } from '@proj-airi/server-sdk'
|
||||
import type { CommonContentPart } from '@xsai/shared-chat'
|
||||
|
||||
import { Client, WebSocketEventSource } from '@proj-airi/server-sdk'
|
||||
import { isStageTamagotchi, isStageWeb } from '@proj-airi/stage-shared'
|
||||
@@ -143,7 +144,7 @@ export const useModsServerChannelStore = defineStore('mods:channels:proj-airi:se
|
||||
}
|
||||
}
|
||||
|
||||
function onContextUpdate(callback: (event: WebSocketBaseEvent<'context:update', ContextUpdate>) => void | Promise<void>) {
|
||||
function onContextUpdate(callback: (event: WebSocketBaseEvent<'context:update', ContextUpdate<Record<string, unknown>, string | CommonContentPart[]>>) => void | Promise<void>) {
|
||||
if (!client.value && !initializing.value)
|
||||
void initialize()
|
||||
|
||||
@@ -168,9 +169,12 @@ export const useModsServerChannelStore = defineStore('mods:channels:proj-airi:se
|
||||
}
|
||||
}
|
||||
|
||||
function sendContextUpdate(message: Omit<ContextUpdate, 'id' | 'contextId'> & Partial<Pick<ContextUpdate, 'id' | 'contextId'>>) {
|
||||
function sendContextUpdate(message: InputContextUpdate) {
|
||||
const id = nanoid()
|
||||
send({ type: 'context:update', data: { id, contextId: id, ...message } })
|
||||
send({
|
||||
type: 'context:update',
|
||||
data: { id, contextId: id, ...message },
|
||||
} as WebSocketEventOptionalSource<string | CommonContentPart[]>)
|
||||
}
|
||||
|
||||
function dispose() {
|
||||
|
||||
@@ -2,4 +2,3 @@ export * from './agents'
|
||||
export * from './orchestrator'
|
||||
export * from './processing-store'
|
||||
export * from './store'
|
||||
export * from './utils/model-catalog'
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { useVisionOrchestratorStore } from './orchestrator'
|
||||
import { useVisionStore } from './store'
|
||||
|
||||
const sendContextUpdate = vi.fn()
|
||||
const runVisionInference = vi.fn()
|
||||
|
||||
vi.mock('vue-i18n', () => ({
|
||||
useI18n: () => ({
|
||||
t: (key: string) => key,
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('../../../composables/vision', () => ({
|
||||
useVisionInference: () => ({
|
||||
lastText: { value: '' },
|
||||
runVisionInference,
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('../../../composables/vision/use-vision-workloads', () => ({
|
||||
getVisionWorkload: (id: string) => ({
|
||||
id,
|
||||
label: `Workload ${id}`,
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('../../mods/api/channel-server', () => ({
|
||||
useModsServerChannelStore: () => ({
|
||||
sendContextUpdate,
|
||||
}),
|
||||
}))
|
||||
|
||||
describe('vision orchestrator', () => {
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia())
|
||||
sendContextUpdate.mockReset()
|
||||
runVisionInference.mockReset()
|
||||
runVisionInference.mockResolvedValue('Frame summary')
|
||||
|
||||
const visionStore = useVisionStore()
|
||||
visionStore.activeProvider = 'mock-provider'
|
||||
visionStore.activeModel = 'mock-model'
|
||||
})
|
||||
|
||||
it('reuses a stable context id for repeated ReplaceSelf updates from the same source', async () => {
|
||||
const store = useVisionOrchestratorStore()
|
||||
|
||||
await store.processCapture({
|
||||
imageDataUrl: 'data:image/jpeg;base64,first',
|
||||
workloadId: 'screen:interpret',
|
||||
sourceId: 'screen:0:0',
|
||||
publishContext: true,
|
||||
})
|
||||
|
||||
await store.processCapture({
|
||||
imageDataUrl: 'data:image/jpeg;base64,second',
|
||||
workloadId: 'screen:interpret',
|
||||
sourceId: 'screen:0:0',
|
||||
publishContext: true,
|
||||
})
|
||||
|
||||
expect(sendContextUpdate).toHaveBeenCalledTimes(2)
|
||||
expect(sendContextUpdate.mock.calls[0]?.[0]).toMatchObject({
|
||||
contextId: 'vision:screen:interpret:screen:0:0',
|
||||
})
|
||||
expect(sendContextUpdate.mock.calls[1]?.[0]).toMatchObject({
|
||||
contextId: 'vision:screen:interpret:screen:0:0',
|
||||
})
|
||||
})
|
||||
|
||||
it('partitions context ids by workload when no source id is available', async () => {
|
||||
const store = useVisionOrchestratorStore()
|
||||
|
||||
await store.processCapture({
|
||||
imageDataUrl: 'data:image/jpeg;base64,first',
|
||||
workloadId: 'screen:interpret',
|
||||
publishContext: true,
|
||||
})
|
||||
|
||||
await store.processCapture({
|
||||
imageDataUrl: 'data:image/jpeg;base64,second',
|
||||
workloadId: 'screen:understand',
|
||||
publishContext: true,
|
||||
})
|
||||
|
||||
expect(sendContextUpdate.mock.calls[0]?.[0]).toMatchObject({
|
||||
contextId: 'vision:screen:interpret',
|
||||
})
|
||||
expect(sendContextUpdate.mock.calls[1]?.[0]).toMatchObject({
|
||||
contextId: 'vision:screen:understand',
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -19,6 +19,12 @@ export interface VisionCapturePayload {
|
||||
publishContext?: boolean
|
||||
}
|
||||
|
||||
function getVisionContextId(payload: Pick<VisionCapturePayload, 'workloadId' | 'sourceId'>) {
|
||||
return payload.sourceId
|
||||
? `vision:${payload.workloadId}:${payload.sourceId}`
|
||||
: `vision:${payload.workloadId}`
|
||||
}
|
||||
|
||||
export const useVisionOrchestratorStore = defineStore('vision-orchestrator', () => {
|
||||
const visionStore = useVisionStore()
|
||||
const { activeProvider, activeModel } = storeToRefs(visionStore)
|
||||
@@ -59,6 +65,7 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', ()
|
||||
|
||||
modsServerChannelStore.sendContextUpdate({
|
||||
strategy: ContextUpdateStrategy.ReplaceSelf,
|
||||
contextId: getVisionContextId(payload),
|
||||
text,
|
||||
content,
|
||||
metadata: {
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
export interface VisionModelInfo {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
tags: string[]
|
||||
recommendedFor: string[]
|
||||
deprecated?: boolean
|
||||
customizable?: boolean
|
||||
}
|
||||
|
||||
export const VISION_MODEL_CATALOG: VisionModelInfo[] = [
|
||||
{
|
||||
id: 'gpt-4o-mini-vision',
|
||||
name: 'GPT-4o mini (Vision)',
|
||||
description: 'Fast and cost-efficient for screen UI understanding and lightweight OCR.',
|
||||
tags: ['fast', 'ui', 'ocr'],
|
||||
recommendedFor: ['UI navigation', 'Quick captions', 'Low-latency feedback'],
|
||||
},
|
||||
{
|
||||
id: 'gpt-4o-vision',
|
||||
name: 'GPT-4o (Vision)',
|
||||
description: 'Stronger reasoning on complex layouts and multi-window scenes.',
|
||||
tags: ['accurate', 'ui', 'reasoning'],
|
||||
recommendedFor: ['Dense UIs', 'Multi-step analysis', 'Ambiguous screens'],
|
||||
},
|
||||
{
|
||||
id: 'claude-3.5-sonnet-vision',
|
||||
name: 'Claude 3.5 Sonnet (Vision)',
|
||||
description: 'Balanced quality for diagrams, docs, and structured UI reading.',
|
||||
tags: ['balanced', 'documents', 'charts'],
|
||||
recommendedFor: ['Docs and dashboards', 'Diagram reading', 'Summaries'],
|
||||
},
|
||||
{
|
||||
id: 'gemini-1.5-pro-vision',
|
||||
name: 'Gemini 1.5 Pro (Vision)',
|
||||
description: 'Great for longer-context screen stories and multi-step tasks.',
|
||||
tags: ['long-context', 'reasoning'],
|
||||
recommendedFor: ['Long sessions', 'Workflow tracking', 'Large screens'],
|
||||
},
|
||||
{
|
||||
id: 'llava-1.6',
|
||||
name: 'LLaVA 1.6 (Local)',
|
||||
description: 'Local-friendly baseline for offline or privacy-focused setups.',
|
||||
tags: ['local', 'offline'],
|
||||
recommendedFor: ['Offline mode', 'On-device evaluation', 'Privacy-first use'],
|
||||
},
|
||||
{
|
||||
id: 'custom',
|
||||
name: 'Custom Model',
|
||||
description: 'Use your own vision model ID from a provider or local runtime.',
|
||||
tags: ['custom'],
|
||||
recommendedFor: ['Self-hosted models', 'Experimental runtimes'],
|
||||
customizable: true,
|
||||
},
|
||||
]
|
||||
Reference in New Issue
Block a user