feat(stage-tamagotchi): added fundamental vision
This commit is contained in:
Neko Ayaka
@@ -38,6 +38,7 @@
|
||||
"@xsai/generate-speech": "catalog:",
|
||||
"@xsai/stream-transcription": "0.4.0-beta.8",
|
||||
"animejs": "^4.3.6",
|
||||
"d3": "catalog:",
|
||||
"dompurify": "^3.3.1",
|
||||
"nanoid": "^5.1.6",
|
||||
"node-vibrant": "^4.0.4",
|
||||
|
||||
@@ -0,0 +1,645 @@
|
||||
<script setup lang="ts">
|
||||
import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
|
||||
import { TimelineRange, TimeSeriesChart } from '@proj-airi/stage-ui/components'
|
||||
import { useChatContextStore } from '@proj-airi/stage-ui/stores/chat/context-store'
|
||||
import { Button, FieldCheckbox, FieldInput, FieldRange, FieldSelect } from '@proj-airi/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { TooltipArrow, TooltipContent, TooltipPortal, TooltipProvider, TooltipRoot, TooltipTrigger } from 'reka-ui'
|
||||
import { computed, ref, watch } from 'vue'
|
||||
|
||||
const chatContextStore = useChatContextStore()
|
||||
const { activeContexts, contextHistory } = storeToRefs(chatContextStore)
|
||||
|
||||
const selectedSource = ref('all')
|
||||
const strategyFilter = ref<'all' | ContextUpdateStrategy>('all')
|
||||
const textFilter = ref('')
|
||||
const onlyWithText = ref(false)
|
||||
const maxHistoryEntries = ref(220)
|
||||
const followLatest = ref(true)
|
||||
const showMetadata = ref(false)
|
||||
const rangeSelection = ref<[number, number] | null>(null)
|
||||
|
||||
const selectedEntryId = ref<string | null>(null)
|
||||
|
||||
const sourceOptions = computed(() => {
|
||||
const entries = Object.keys(activeContexts.value).sort()
|
||||
return [
|
||||
{ label: 'All modules', value: 'all' },
|
||||
...entries.map(source => ({ label: source, value: source })),
|
||||
]
|
||||
})
|
||||
|
||||
const strategyOptions = [
|
||||
{ label: 'All strategies', value: 'all' },
|
||||
{ label: 'Replace', value: ContextUpdateStrategy.ReplaceSelf },
|
||||
{ label: 'Append', value: ContextUpdateStrategy.AppendSelf },
|
||||
]
|
||||
|
||||
const baseHistory = computed(() => {
|
||||
const query = textFilter.value.trim().toLowerCase()
|
||||
|
||||
return contextHistory.value
|
||||
.filter((entry) => {
|
||||
if (selectedSource.value === 'all')
|
||||
return true
|
||||
return entry.sourceKey === selectedSource.value
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (strategyFilter.value === 'all')
|
||||
return true
|
||||
return entry.strategy === strategyFilter.value
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (!onlyWithText.value)
|
||||
return true
|
||||
return Boolean(entry.text?.trim())
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (!query)
|
||||
return true
|
||||
return [
|
||||
entry.sourceKey,
|
||||
entry.text,
|
||||
entry.id,
|
||||
entry.contextId,
|
||||
].filter(Boolean).join(' ').toLowerCase().includes(query)
|
||||
})
|
||||
.slice(-maxHistoryEntries.value)
|
||||
})
|
||||
|
||||
const filteredHistory = computed(() => {
|
||||
if (!rangeSelection.value)
|
||||
return baseHistory.value
|
||||
|
||||
const [start, end] = rangeSelection.value
|
||||
return baseHistory.value.filter((entry) => {
|
||||
const timestamp = entry.createdAt ?? 0
|
||||
return timestamp >= start && timestamp <= end
|
||||
})
|
||||
})
|
||||
|
||||
const rangeData = computed(() => baseHistory.value.map(entry => entry.createdAt ?? 0))
|
||||
|
||||
const activeContextGroups = computed(() => {
|
||||
const keys = selectedSource.value === 'all'
|
||||
? Object.keys(activeContexts.value)
|
||||
: [selectedSource.value]
|
||||
|
||||
return keys
|
||||
.sort()
|
||||
.map((key) => {
|
||||
const contexts = activeContexts.value[key] ?? []
|
||||
const latest = contexts.at(-1)?.createdAt ?? 0
|
||||
return {
|
||||
key,
|
||||
contexts,
|
||||
latest,
|
||||
}
|
||||
})
|
||||
.sort((a, b) => b.latest - a.latest)
|
||||
})
|
||||
|
||||
const totalActiveContexts = computed(() => {
|
||||
return Object.values(activeContexts.value).reduce((sum, contexts) => sum + contexts.length, 0)
|
||||
})
|
||||
|
||||
const updateWindowMs = 60_000
|
||||
const updateBucketCount = 30
|
||||
const updateBucketMs = updateWindowMs / updateBucketCount
|
||||
|
||||
const updateTimeline = computed(() => {
|
||||
const now = Date.now()
|
||||
const buckets = Array.from({ length: updateBucketCount }, () => 0)
|
||||
|
||||
for (const entry of filteredHistory.value) {
|
||||
const ts = entry.createdAt || now
|
||||
const offset = ts - (now - updateWindowMs)
|
||||
if (offset < 0 || offset > updateWindowMs)
|
||||
continue
|
||||
const index = Math.min(updateBucketCount - 1, Math.max(0, Math.floor(offset / updateBucketMs)))
|
||||
buckets[index] += 1
|
||||
}
|
||||
|
||||
const peak = Math.max(1, ...buckets)
|
||||
const normalized = buckets.map(value => value / peak)
|
||||
return {
|
||||
normalized,
|
||||
currentValue: normalized[normalized.length - 1] ?? 0,
|
||||
total: buckets.reduce((sum, value) => sum + value, 0),
|
||||
}
|
||||
})
|
||||
|
||||
const queueEntries = computed(() => filteredHistory.value.slice(-80))
|
||||
|
||||
const selectedEntry = computed(() => {
|
||||
if (!selectedEntryId.value)
|
||||
return null
|
||||
return filteredHistory.value.find(entry => entry.id === selectedEntryId.value) || null
|
||||
})
|
||||
|
||||
const activeFilterSummary = computed(() => {
|
||||
const filters = []
|
||||
if (selectedSource.value !== 'all')
|
||||
filters.push(`module:${selectedSource.value}`)
|
||||
if (strategyFilter.value !== 'all')
|
||||
filters.push(`strategy:${formatStrategy(strategyFilter.value)}`)
|
||||
if (onlyWithText.value)
|
||||
filters.push('text-only')
|
||||
if (textFilter.value.trim())
|
||||
filters.push(`query:${textFilter.value.trim()}`)
|
||||
if (rangeSelection.value) {
|
||||
const [start, end] = rangeSelection.value
|
||||
filters.push(`${new Date(start).toLocaleTimeString()} → ${new Date(end).toLocaleTimeString()}`)
|
||||
}
|
||||
return filters.length ? filters.join(' · ') : 'All updates'
|
||||
})
|
||||
|
||||
function formatStrategy(strategy: ContextUpdateStrategy) {
|
||||
return strategy === ContextUpdateStrategy.AppendSelf ? 'Append' : 'Replace'
|
||||
}
|
||||
|
||||
function formatTimestamp(timestamp: number) {
|
||||
return new Date(timestamp).toLocaleTimeString()
|
||||
}
|
||||
|
||||
function formatRelativeTime(timestamp: number | null | undefined) {
|
||||
if (!timestamp)
|
||||
return 'Never'
|
||||
const diffMs = Date.now() - timestamp
|
||||
const diffSeconds = Math.max(0, Math.floor(diffMs / 1000))
|
||||
if (diffSeconds < 60)
|
||||
return `${diffSeconds}s ago`
|
||||
const diffMinutes = Math.floor(diffSeconds / 60)
|
||||
if (diffMinutes < 60)
|
||||
return `${diffMinutes}m ago`
|
||||
const diffHours = Math.floor(diffMinutes / 60)
|
||||
return `${diffHours}h ago`
|
||||
}
|
||||
|
||||
function summarizeText(text: string, limit = 120) {
|
||||
if (!text)
|
||||
return 'No context text.'
|
||||
if (text.length <= limit)
|
||||
return text
|
||||
return `${text.slice(0, limit)}…`
|
||||
}
|
||||
|
||||
function queueDotClass(entry: { strategy: ContextUpdateStrategy }) {
|
||||
if (entry.strategy === ContextUpdateStrategy.AppendSelf)
|
||||
return 'bg-emerald-400/80'
|
||||
return 'bg-amber-400/80'
|
||||
}
|
||||
|
||||
function resetRange() {
|
||||
rangeSelection.value = null
|
||||
}
|
||||
|
||||
watch(() => filteredHistory.value.length, () => {
|
||||
if (!followLatest.value)
|
||||
return
|
||||
const latest = filteredHistory.value.at(-1)
|
||||
if (latest)
|
||||
selectedEntryId.value = latest.id
|
||||
})
|
||||
|
||||
watch([selectedSource, strategyFilter, textFilter, onlyWithText], () => {
|
||||
if (!followLatest.value)
|
||||
return
|
||||
const latest = filteredHistory.value.at(-1)
|
||||
if (latest)
|
||||
selectedEntryId.value = latest.id
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-gradient-to-br',
|
||||
'from-neutral-100',
|
||||
'via-white',
|
||||
'to-primary-100/50',
|
||||
'p-5',
|
||||
'shadow-sm',
|
||||
'dark:from-neutral-900/80',
|
||||
'dark:via-neutral-950',
|
||||
'dark:to-primary-900/20',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'flex-col', 'gap-5', 'lg:flex-row', 'lg:items-center', 'lg:justify-between']">
|
||||
<div :class="['flex', 'flex-col', 'gap-2']">
|
||||
<div :class="['text-xs', 'uppercase', 'tracking-[0.25em]', 'text-neutral-400']">
|
||||
Context Observatory
|
||||
</div>
|
||||
<div :class="['text-2xl', 'font-semibold', 'text-neutral-800', 'dark:text-neutral-100']">
|
||||
Active contexts: {{ totalActiveContexts }}
|
||||
</div>
|
||||
<div :class="['text-sm', 'text-neutral-500', 'dark:text-neutral-400']">
|
||||
{{ activeFilterSummary }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['grid', 'gap-3', 'md:grid-cols-2', 'xl:grid-cols-4']">
|
||||
<FieldSelect
|
||||
v-model="selectedSource"
|
||||
label="Module"
|
||||
:options="sourceOptions"
|
||||
/>
|
||||
<FieldSelect
|
||||
v-model="strategyFilter"
|
||||
label="Strategy"
|
||||
:options="strategyOptions"
|
||||
/>
|
||||
<FieldInput
|
||||
v-model="textFilter"
|
||||
label="Search"
|
||||
placeholder="Find by text / id"
|
||||
/>
|
||||
<FieldRange
|
||||
v-model="maxHistoryEntries"
|
||||
label="Log depth"
|
||||
:min="40"
|
||||
:max="300"
|
||||
:step="20"
|
||||
:format-value="value => `${value} entries`"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-4', 'grid', 'gap-4', 'md:grid-cols-3']">
|
||||
<FieldCheckbox
|
||||
v-model="onlyWithText"
|
||||
label="Text-only updates"
|
||||
description="Hide empty or metadata-only updates."
|
||||
/>
|
||||
<FieldCheckbox
|
||||
v-model="followLatest"
|
||||
label="Follow latest"
|
||||
description="Auto-select newest update."
|
||||
/>
|
||||
<FieldCheckbox
|
||||
v-model="showMetadata"
|
||||
label="Show metadata"
|
||||
description="Display raw metadata in details."
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['grid', 'gap-6', 'xl:grid-cols-[minmax(0,1.1fr)_minmax(0,0.9fr)]']">
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Cadence
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ updateTimeline.total }} updates / min
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<TimeSeriesChart
|
||||
:is-active="false"
|
||||
:history="updateTimeline.normalized"
|
||||
:current-value="updateTimeline.currentValue"
|
||||
:show-header="false"
|
||||
:show-legend="false"
|
||||
:show-current-value="false"
|
||||
:show-area="true"
|
||||
:show-active-indicator="false"
|
||||
:height="96"
|
||||
unit="/min"
|
||||
:precision="0"
|
||||
/>
|
||||
|
||||
<div :class="['mt-4', 'flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Timeline range
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="secondary"
|
||||
:disabled="!rangeSelection"
|
||||
:label="rangeSelection ? 'Reset range' : 'Full range'"
|
||||
@click="resetRange"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<TimelineRange
|
||||
v-model:range="rangeSelection"
|
||||
:data="rangeData"
|
||||
:height="140"
|
||||
:bins="48"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Active contexts
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ activeContextGroups.length }} modules
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-3', 'grid', 'gap-3', 'lg:grid-cols-2']">
|
||||
<div
|
||||
v-for="group in activeContextGroups"
|
||||
:key="group.key"
|
||||
:class="[
|
||||
'rounded-xl',
|
||||
'border',
|
||||
'border-neutral-200',
|
||||
'bg-white',
|
||||
'p-3',
|
||||
'dark:border-neutral-800',
|
||||
'dark:bg-neutral-900',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between', 'gap-2']">
|
||||
<div :class="['text-sm', 'font-semibold', 'text-neutral-700', 'dark:text-neutral-200']">
|
||||
{{ group.key }}
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ group.contexts.length }} contexts · {{ formatRelativeTime(group.latest) }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="group.contexts.length === 0" :class="['mt-2', 'text-xs', 'text-neutral-400']">
|
||||
No active contexts.
|
||||
</div>
|
||||
|
||||
<div v-else :class="['mt-3', 'max-h-60', 'space-y-3', 'overflow-y-auto', 'pr-1']">
|
||||
<div
|
||||
v-for="context in group.contexts.slice().reverse()"
|
||||
:key="context.id"
|
||||
:class="[
|
||||
'rounded-lg',
|
||||
'border',
|
||||
'border-neutral-100',
|
||||
'bg-neutral-50',
|
||||
'p-3',
|
||||
'text-sm',
|
||||
'text-neutral-700',
|
||||
'dark:border-neutral-800/60',
|
||||
'dark:bg-neutral-950/60',
|
||||
'dark:text-neutral-200',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between', 'text-xs', 'text-neutral-400']">
|
||||
<span>{{ formatStrategy(context.strategy) }}</span>
|
||||
<span>{{ context.createdAt ? formatTimestamp(context.createdAt) : 'Unknown' }}</span>
|
||||
</div>
|
||||
<div :class="['mt-2', 'whitespace-pre-wrap', 'text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||
{{ context.text || 'No context text.' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-if="activeContextGroups.length === 0"
|
||||
:class="[
|
||||
'rounded-xl',
|
||||
'border-2',
|
||||
'border-dashed',
|
||||
'border-neutral-200',
|
||||
'px-4',
|
||||
'py-10',
|
||||
'text-sm',
|
||||
'text-neutral-400',
|
||||
'dark:border-neutral-800',
|
||||
]"
|
||||
>
|
||||
No active context modules yet.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Selected update
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ selectedEntry ? formatTimestamp(selectedEntry.createdAt || Date.now()) : 'None' }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="selectedEntry" :class="['mt-3', 'space-y-3']">
|
||||
<div :class="['flex', 'items-center', 'gap-2', 'text-xs', 'text-neutral-400']">
|
||||
<span>{{ selectedEntry.sourceKey }}</span>
|
||||
<span>·</span>
|
||||
<span>{{ formatStrategy(selectedEntry.strategy) }}</span>
|
||||
<span>·</span>
|
||||
<span>{{ selectedEntry.contextId }}</span>
|
||||
</div>
|
||||
<div :class="['whitespace-pre-wrap', 'rounded-lg', 'bg-white', 'p-3', 'text-sm', 'text-neutral-700', 'dark:bg-neutral-900', 'dark:text-neutral-200']">
|
||||
{{ selectedEntry.text || 'No context text.' }}
|
||||
</div>
|
||||
|
||||
<pre
|
||||
v-if="showMetadata"
|
||||
:class="[
|
||||
'rounded-lg',
|
||||
'bg-neutral-900',
|
||||
'p-3',
|
||||
'text-xs',
|
||||
'text-neutral-100',
|
||||
'overflow-auto',
|
||||
]"
|
||||
>
|
||||
{{ JSON.stringify(selectedEntry.metadata ?? {}, null, 2) }}
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
<div v-else :class="['mt-3', 'text-sm', 'text-neutral-400']">
|
||||
Pick a queue dot or stream block to inspect.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
:class="[
|
||||
'rounded-2xl',
|
||||
'bg-neutral-100',
|
||||
'p-4',
|
||||
'shadow-sm',
|
||||
'dark:bg-[rgba(0,0,0,0.35)]',
|
||||
]"
|
||||
>
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['text-sm', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Stream blocks
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
{{ queueEntries.length }} recent updates
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-4', 'grid', 'grid-cols-6', 'gap-2', 'sm:grid-cols-8']">
|
||||
<TooltipProvider v-for="entry in queueEntries" :key="entry.id" :delay-duration="120">
|
||||
<TooltipRoot>
|
||||
<TooltipTrigger as-child>
|
||||
<button
|
||||
type="button"
|
||||
:class="[
|
||||
'h-9',
|
||||
'w-9',
|
||||
'rounded-lg',
|
||||
'border',
|
||||
'border-neutral-200',
|
||||
'bg-white',
|
||||
'transition',
|
||||
'duration-150',
|
||||
'dark:border-neutral-800',
|
||||
'dark:bg-neutral-900',
|
||||
selectedEntryId === entry.id ? 'ring-2 ring-primary-500/70' : 'hover:border-neutral-300',
|
||||
]"
|
||||
@click="selectedEntryId = entry.id"
|
||||
>
|
||||
<span :class="['block', 'h-full', 'w-full', 'rounded-lg', queueDotClass(entry)]" />
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipPortal>
|
||||
<TooltipContent
|
||||
:class="[
|
||||
'rounded-md',
|
||||
'bg-white',
|
||||
'px-2',
|
||||
'py-1',
|
||||
'text-xs',
|
||||
'text-neutral-700',
|
||||
'shadow-md',
|
||||
'dark:bg-neutral-900',
|
||||
'dark:text-neutral-200',
|
||||
]"
|
||||
>
|
||||
<div>{{ entry.sourceKey }} · {{ formatStrategy(entry.strategy) }}</div>
|
||||
<div>{{ entry.createdAt ? formatTimestamp(entry.createdAt) : 'Unknown' }}</div>
|
||||
<div>{{ summarizeText(entry.text, 80) }}</div>
|
||||
<TooltipArrow :class="['fill-white', 'dark:fill-neutral-900']" />
|
||||
</TooltipContent>
|
||||
</TooltipPortal>
|
||||
</TooltipRoot>
|
||||
</TooltipProvider>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-4', 'flex', 'flex-col', 'gap-2']">
|
||||
<button
|
||||
v-for="entry in filteredHistory.slice().reverse().slice(0, 12)"
|
||||
:key="entry.id"
|
||||
type="button"
|
||||
:class="[
|
||||
'flex',
|
||||
'items-start',
|
||||
'justify-between',
|
||||
'gap-3',
|
||||
'rounded-xl',
|
||||
'border',
|
||||
'border-neutral-200',
|
||||
'bg-white',
|
||||
'p-3',
|
||||
'text-left',
|
||||
'transition',
|
||||
'duration-150',
|
||||
selectedEntryId === entry.id ? 'border-primary-400/70 shadow-sm' : 'hover:border-neutral-300',
|
||||
'dark:border-neutral-800',
|
||||
'dark:bg-neutral-900',
|
||||
'dark:hover:border-neutral-700',
|
||||
]"
|
||||
@click="selectedEntryId = entry.id"
|
||||
>
|
||||
<div :class="['flex', 'flex-col', 'gap-1']">
|
||||
<div :class="['flex', 'items-center', 'gap-2']">
|
||||
<span :class="['h-2.5', 'w-2.5', 'rounded-full', queueDotClass(entry)]" />
|
||||
<span :class="['text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
{{ entry.sourceKey }}
|
||||
</span>
|
||||
<span :class="['text-xs', 'text-neutral-400']">
|
||||
{{ formatStrategy(entry.strategy) }}
|
||||
</span>
|
||||
</div>
|
||||
<div :class="['text-sm', 'text-neutral-700', 'dark:text-neutral-200']">
|
||||
{{ summarizeText(entry.text, 120) }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<TooltipProvider :delay-duration="150">
|
||||
<TooltipRoot>
|
||||
<TooltipTrigger as-child>
|
||||
<div :class="['text-xs', 'text-neutral-400', 'cursor-help']">
|
||||
{{ entry.createdAt ? formatTimestamp(entry.createdAt) : 'Unknown' }}
|
||||
</div>
|
||||
</TooltipTrigger>
|
||||
<TooltipPortal>
|
||||
<TooltipContent
|
||||
:class="[
|
||||
'rounded-md',
|
||||
'bg-white',
|
||||
'px-2',
|
||||
'py-1',
|
||||
'text-xs',
|
||||
'text-neutral-700',
|
||||
'shadow-md',
|
||||
'dark:bg-neutral-900',
|
||||
'dark:text-neutral-200',
|
||||
]"
|
||||
>
|
||||
{{ entry.createdAt ? new Date(entry.createdAt).toLocaleString() : 'Unknown timestamp' }}
|
||||
<TooltipArrow :class="['fill-white', 'dark:fill-neutral-900']" />
|
||||
</TooltipContent>
|
||||
</TooltipPortal>
|
||||
</TooltipRoot>
|
||||
</TooltipProvider>
|
||||
</button>
|
||||
|
||||
<div
|
||||
v-if="filteredHistory.length === 0"
|
||||
:class="[
|
||||
'rounded-xl',
|
||||
'border-2',
|
||||
'border-dashed',
|
||||
'border-neutral-200',
|
||||
'px-4',
|
||||
'py-10',
|
||||
'text-center',
|
||||
'text-sm',
|
||||
'text-neutral-400',
|
||||
'dark:border-neutral-800',
|
||||
]"
|
||||
>
|
||||
No updates yet. Emit a context update to populate the stream.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,9 +1,387 @@
|
||||
<script setup lang="ts">
|
||||
import { WIP } from '@proj-airi/stage-ui/components'
|
||||
import { Alert, ErrorContainer, RadioCardManySelect, RadioCardSimple } from '@proj-airi/stage-ui/components'
|
||||
import { useAnalytics } from '@proj-airi/stage-ui/composables'
|
||||
import { useVisionProcessingStore, useVisionStore } from '@proj-airi/stage-ui/stores/modules/vision'
|
||||
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
|
||||
import { FieldRange } from '@proj-airi/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { computed, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { RouterLink } from 'vue-router'
|
||||
|
||||
const providersStore = useProvidersStore()
|
||||
const visionStore = useVisionStore()
|
||||
const visionProcessingStore = useVisionProcessingStore()
|
||||
const { persistedChatProvidersMetadata, configuredProviders } = storeToRefs(providersStore)
|
||||
const {
|
||||
activeProvider,
|
||||
activeModel,
|
||||
customModelName,
|
||||
modelSearchQuery,
|
||||
supportsModelListing,
|
||||
providerModels,
|
||||
isLoadingActiveProviderModels,
|
||||
activeProviderModelError,
|
||||
} = storeToRefs(visionStore)
|
||||
const {
|
||||
captureIntervalMs,
|
||||
captureCount,
|
||||
contextUpdateCount,
|
||||
lastCaptureAt,
|
||||
lastContextUpdateAt,
|
||||
isRunning,
|
||||
} = storeToRefs(visionProcessingStore)
|
||||
|
||||
const { t } = useI18n()
|
||||
const { trackProviderClick } = useAnalytics()
|
||||
|
||||
watch(activeProvider, async (provider) => {
|
||||
if (!provider)
|
||||
return
|
||||
await visionStore.loadModelsForProvider(provider)
|
||||
}, { immediate: true })
|
||||
|
||||
function updateCustomModelName(value: string) {
|
||||
customModelName.value = value
|
||||
}
|
||||
|
||||
function handleDeleteProvider(providerId: string) {
|
||||
if (activeProvider.value === providerId) {
|
||||
activeProvider.value = ''
|
||||
activeModel.value = ''
|
||||
}
|
||||
providersStore.deleteProvider(providerId)
|
||||
}
|
||||
|
||||
const formattedLastCapture = computed(() => formatRelativeTime(lastCaptureAt.value))
|
||||
const formattedLastContextUpdate = computed(() => formatRelativeTime(lastContextUpdateAt.value))
|
||||
|
||||
function formatRelativeTime(timestamp: number | null) {
|
||||
if (!timestamp)
|
||||
return 'Never'
|
||||
|
||||
const diffMs = Date.now() - timestamp
|
||||
const diffSeconds = Math.max(0, Math.floor(diffMs / 1000))
|
||||
if (diffSeconds < 60)
|
||||
return `${diffSeconds}s ago`
|
||||
const diffMinutes = Math.floor(diffSeconds / 60)
|
||||
if (diffMinutes < 60)
|
||||
return `${diffMinutes}m ago`
|
||||
const diffHours = Math.floor(diffMinutes / 60)
|
||||
return `${diffHours}h ago`
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<WIP />
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<div :class="['rounded-xl', 'bg-neutral-50', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
|
||||
<div :class="['flex', 'flex-col', 'gap-4']">
|
||||
<div>
|
||||
<h2 :class="['text-lg', 'text-neutral-500', 'md:text-2xl', 'dark:text-neutral-500']">
|
||||
{{ t('settings.pages.providers.title') }}
|
||||
</h2>
|
||||
<div :class="['text-neutral-400', 'dark:text-neutral-400']">
|
||||
<span>{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.description') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div :class="['max-w-full']">
|
||||
<fieldset
|
||||
v-if="persistedChatProvidersMetadata.length > 0"
|
||||
:class="['flex', 'min-w-0', 'flex-row', 'gap-4', 'of-x-scroll', 'scroll-smooth']"
|
||||
:style="{ 'scrollbar-width': 'none' }"
|
||||
role="radiogroup"
|
||||
>
|
||||
<RadioCardSimple
|
||||
v-for="metadata in persistedChatProvidersMetadata"
|
||||
:id="metadata.id"
|
||||
:key="metadata.id"
|
||||
v-model="activeProvider"
|
||||
name="provider"
|
||||
:value="metadata.id"
|
||||
:title="metadata.localizedName || 'Unknown'"
|
||||
:description="metadata.localizedDescription"
|
||||
@click="trackProviderClick(metadata.id, 'vision')"
|
||||
>
|
||||
<template #topRight>
|
||||
<button
|
||||
type="button"
|
||||
:class="[
|
||||
'rounded',
|
||||
'bg-neutral-100',
|
||||
'p-1',
|
||||
'text-neutral-600',
|
||||
'transition-colors',
|
||||
'hover:bg-neutral-200',
|
||||
'dark:bg-neutral-800/60',
|
||||
'dark:text-neutral-300',
|
||||
'dark:hover:bg-neutral-700/60',
|
||||
]"
|
||||
@click.stop.prevent="handleDeleteProvider(metadata.id)"
|
||||
>
|
||||
<div :class="['text-base', 'i-solar:trash-bin-trash-bold-duotone']" />
|
||||
</button>
|
||||
</template>
|
||||
|
||||
<template v-if="configuredProviders[metadata.id] === false" #bottomRight>
|
||||
<div
|
||||
:class="[
|
||||
'rounded',
|
||||
'bg-amber-100',
|
||||
'px-2',
|
||||
'py-0.5',
|
||||
'text-xs',
|
||||
'font-medium',
|
||||
'text-amber-700',
|
||||
'dark:bg-amber-900/30',
|
||||
'dark:text-amber-300',
|
||||
]"
|
||||
>
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.health_check_failed') }}
|
||||
</div>
|
||||
</template>
|
||||
</RadioCardSimple>
|
||||
<RouterLink
|
||||
to="/settings/providers"
|
||||
:class="[
|
||||
'relative',
|
||||
'min-w-50',
|
||||
'w-fit',
|
||||
'rounded-xl',
|
||||
'border-2',
|
||||
'border-neutral-100',
|
||||
'bg-white',
|
||||
'p-4',
|
||||
'transition',
|
||||
'duration-200',
|
||||
'ease-in-out',
|
||||
'hover:border-primary-500/30',
|
||||
'dark:border-neutral-900',
|
||||
'dark:bg-neutral-900/20',
|
||||
'dark:hover:border-primary-400/30',
|
||||
'flex',
|
||||
'items-center',
|
||||
'justify-center',
|
||||
]"
|
||||
>
|
||||
<div :class="['text-2xl', 'text-neutral-500', 'dark:text-neutral-500', 'i-solar:add-circle-line-duotone']" />
|
||||
<div
|
||||
:class="['absolute', 'inset-0', 'z--1', 'bg-dotted-neutral-200/80', 'dark:bg-dotted-neutral-700/50']"
|
||||
:style="{ 'background-size': '10px 10px', 'mask-image': 'linear-gradient(165deg, white 30%, transparent 50%)' }"
|
||||
/>
|
||||
</RouterLink>
|
||||
</fieldset>
|
||||
<div v-else>
|
||||
<RouterLink
|
||||
to="/settings/providers"
|
||||
:class="[
|
||||
'flex',
|
||||
'items-center',
|
||||
'gap-3',
|
||||
'rounded-lg',
|
||||
'border-2',
|
||||
'border-dashed',
|
||||
'border-neutral-200',
|
||||
'bg-neutral-50',
|
||||
'p-4',
|
||||
'transition',
|
||||
'duration-200',
|
||||
'ease-in-out',
|
||||
'dark:border-neutral-800',
|
||||
'dark:bg-neutral-800',
|
||||
]"
|
||||
>
|
||||
<div :class="['text-2xl', 'text-amber-500', 'dark:text-amber-400', 'i-solar:warning-circle-line-duotone']" />
|
||||
<div :class="['flex', 'flex-col']">
|
||||
<span :class="['font-medium']">
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.no_providers_configured_title') }}
|
||||
</span>
|
||||
<span :class="['text-sm', 'text-neutral-400', 'dark:text-neutral-500']">
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.no_providers_configured_description') }}
|
||||
</span>
|
||||
</div>
|
||||
<div :class="['ml-auto', 'text-xl', 'text-neutral-400', 'dark:text-neutral-500', 'i-solar:arrow-right-line-duotone']" />
|
||||
</RouterLink>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="activeProvider && supportsModelListing" :class="['rounded-xl', 'bg-neutral-50', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
|
||||
<div :class="['flex', 'flex-col', 'gap-4']">
|
||||
<div>
|
||||
<h2 :class="['text-lg', 'md:text-2xl']">
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.title') }}
|
||||
</h2>
|
||||
<div :class="['text-neutral-400', 'dark:text-neutral-400']">
|
||||
<span>{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.subtitle') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="isLoadingActiveProviderModels" :class="['flex', 'items-center', 'justify-center', 'py-4']">
|
||||
<div :class="['mr-2', 'animate-spin']">
|
||||
<div :class="['text-xl', 'i-solar:spinner-line-duotone']" />
|
||||
</div>
|
||||
<span>{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.loading') }}</span>
|
||||
</div>
|
||||
|
||||
<ErrorContainer
|
||||
v-else-if="activeProviderModelError"
|
||||
:title="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.error')"
|
||||
:error="activeProviderModelError"
|
||||
/>
|
||||
|
||||
<Alert
|
||||
v-else-if="providerModels.length === 0 && !isLoadingActiveProviderModels"
|
||||
type="warning"
|
||||
>
|
||||
<template #title>
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.no_models') }}
|
||||
</template>
|
||||
<template #content>
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.no_models_description') }}
|
||||
</template>
|
||||
</Alert>
|
||||
|
||||
<template v-else-if="providerModels.length > 0">
|
||||
<RadioCardManySelect
|
||||
v-model="activeModel"
|
||||
v-model:search-query="modelSearchQuery"
|
||||
:items="providerModels.sort((a, b) => a.id === activeModel ? -1 : b.id === activeModel ? 1 : 0)"
|
||||
:searchable="true"
|
||||
:search-placeholder="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.search_placeholder')"
|
||||
:search-no-results-title="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.no_search_results')"
|
||||
:search-no-results-description="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.no_search_results_description', { query: modelSearchQuery })"
|
||||
:search-results-text="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.search_results', { count: '{count}', total: '{total}' })"
|
||||
:custom-input-placeholder="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.custom_model_placeholder')"
|
||||
:expand-button-text="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.expand')"
|
||||
:collapse-button-text="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.collapse')"
|
||||
@update:custom-value="updateCustomModelName"
|
||||
/>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="activeProvider && !supportsModelListing" :class="['rounded-xl', 'bg-neutral-50', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
|
||||
<div :class="['flex', 'flex-col', 'gap-4']">
|
||||
<div>
|
||||
<h2 :class="['text-lg', 'text-neutral-500', 'md:text-2xl', 'dark:text-neutral-400']">
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.title') }}
|
||||
</h2>
|
||||
<div :class="['text-neutral-400', 'dark:text-neutral-500']">
|
||||
<span>{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.subtitle') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
:class="[
|
||||
'flex',
|
||||
'items-center',
|
||||
'gap-3',
|
||||
'rounded-lg',
|
||||
'border',
|
||||
'border-primary-200',
|
||||
'bg-primary-50',
|
||||
'p-4',
|
||||
'dark:border-primary-800',
|
||||
'dark:bg-primary-900/20',
|
||||
]"
|
||||
>
|
||||
<div :class="['text-2xl', 'text-primary-500', 'dark:text-primary-400', 'i-solar:info-circle-line-duotone']" />
|
||||
<div :class="['flex', 'flex-col']">
|
||||
<span :class="['font-medium']">
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.not_supported') }}
|
||||
</span>
|
||||
<span :class="['text-sm', 'text-primary-600', 'dark:text-primary-400']">
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.not_supported_description') }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['mt-2']">
|
||||
<label :class="['mb-1', 'block', 'text-sm', 'font-medium']">
|
||||
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.manual_model_name') }}
|
||||
</label>
|
||||
<input
|
||||
v-model="activeModel"
|
||||
type="text"
|
||||
:class="[
|
||||
'w-full',
|
||||
'rounded',
|
||||
'border',
|
||||
'border-neutral-300',
|
||||
'bg-white',
|
||||
'px-3',
|
||||
'py-2',
|
||||
'dark:border-neutral-700',
|
||||
'dark:bg-neutral-900',
|
||||
]"
|
||||
:placeholder="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.manual_model_placeholder')"
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['rounded-xl', 'bg-neutral-50', 'p-4', 'dark:bg-[rgba(0,0,0,0.3)]']">
|
||||
<div :class="['flex', 'flex-col', 'gap-4']">
|
||||
<div>
|
||||
<h2 :class="['text-lg', 'text-neutral-500', 'md:text-2xl', 'dark:text-neutral-400']">
|
||||
Vision capture cadence
|
||||
</h2>
|
||||
<div :class="['text-neutral-400', 'dark:text-neutral-400']">
|
||||
Tune how frequently the vision ticker captures a frame.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<FieldRange
|
||||
v-model="captureIntervalMs"
|
||||
label="Capture interval"
|
||||
description="Lower values capture more frequently and may increase resource use."
|
||||
:min="500"
|
||||
:max="15000"
|
||||
:step="250"
|
||||
:format-value="value => `${(value / 1000).toFixed(2)}s`"
|
||||
/>
|
||||
|
||||
<div :class="['grid', 'gap-4', 'md:grid-cols-3']">
|
||||
<div :class="['rounded-lg', 'border', 'border-neutral-200', 'bg-white', 'p-3', 'dark:border-neutral-800', 'dark:bg-neutral-900']">
|
||||
<div :class="['text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Ticker
|
||||
</div>
|
||||
<div :class="['text-sm', 'font-medium', 'text-neutral-600', 'dark:text-neutral-200']">
|
||||
{{ isRunning ? 'Active' : 'Idle' }}
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
Last capture {{ formattedLastCapture }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['rounded-lg', 'border', 'border-neutral-200', 'bg-white', 'p-3', 'dark:border-neutral-800', 'dark:bg-neutral-900']">
|
||||
<div :class="['text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Captures
|
||||
</div>
|
||||
<div :class="['text-sm', 'font-medium', 'text-neutral-600', 'dark:text-neutral-200']">
|
||||
{{ captureCount }}
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
Last update {{ formattedLastCapture }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div :class="['rounded-lg', 'border', 'border-neutral-200', 'bg-white', 'p-3', 'dark:border-neutral-800', 'dark:bg-neutral-900']">
|
||||
<div :class="['text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
Context updates
|
||||
</div>
|
||||
<div :class="['text-sm', 'font-medium', 'text-neutral-600', 'dark:text-neutral-200']">
|
||||
{{ contextUpdateCount }}
|
||||
</div>
|
||||
<div :class="['text-xs', 'text-neutral-400']">
|
||||
Last update {{ formattedLastContextUpdate }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<route lang="yaml">
|
||||
@@ -13,5 +391,4 @@ meta:
|
||||
subtitleKey: settings.title
|
||||
stageTransition:
|
||||
name: slide
|
||||
pageSpecificAvailable: true
|
||||
</route>
|
||||
|
||||
@@ -67,7 +67,7 @@ useScrollToHash(() => route.hash, {
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div mb-6 flex flex-col gap-5>
|
||||
<div :class="['mb-6', 'flex', 'flex-col', 'gap-5', 'pb-10']">
|
||||
<div bg="primary-500/10 dark:primary-800/25" rounded-lg p-4>
|
||||
<div mb-2 text-xl font-normal text="primary-800 dark:primary-100">
|
||||
{{ $t('settings.pages.providers.helpinfo.title') }}
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
"./stores/analytics": "./src/stores/analytics/index.ts",
|
||||
"./stores/character": "./src/stores/character/index.ts",
|
||||
"./stores/settings": "./src/stores/settings/index.ts",
|
||||
"./stores/modules/vision": "./src/stores/modules/vision/index.ts",
|
||||
"./stores/*": "./src/stores/*.ts",
|
||||
"./stores": "./src/stores/index.ts",
|
||||
"./workers/vad": "./src/workers/vad/index.ts",
|
||||
@@ -93,6 +94,7 @@
|
||||
"animejs": "^4.3.6",
|
||||
"better-auth": "catalog:",
|
||||
"culori": "^4.0.2",
|
||||
"d3": "catalog:",
|
||||
"date-fns": "^4.1.0",
|
||||
"dompurify": "^3.3.1",
|
||||
"es-toolkit": "catalog:",
|
||||
@@ -160,6 +162,7 @@
|
||||
"@proj-airi/vite-plugin-warpdrive": "workspace:*",
|
||||
"@types/audioworklet": "catalog:",
|
||||
"@types/culori": "^4.0.1",
|
||||
"@types/d3": "catalog:",
|
||||
"@types/hast": "catalog:",
|
||||
"@types/splitpanes": "catalog:",
|
||||
"@types/unist": "catalog:",
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
export { default as AudioSpectrumVisualizer } from './audio-spectrum-visualizer.vue'
|
||||
export { default as AudioSpectrum } from './audio-spectrum.vue'
|
||||
export { default as LevelMeter } from './level-meter.vue'
|
||||
export { default as ProcessingMeter } from './processing-meter.vue'
|
||||
export { default as TestDummyMarker } from './test-dummy-marker.vue'
|
||||
export { default as ThresholdMeter } from './threshold-meter.vue'
|
||||
export { default as TimeSeriesChart } from './time-series-chart.vue'
|
||||
export { default as TimelineRange } from './timeline-range.vue'
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import ProcessingMeter from './processing-meter.vue'
|
||||
|
||||
const processingHistory = ref<number[]>([])
|
||||
const processingValue = ref(0)
|
||||
const contextUpdatesPerMinute = ref(12)
|
||||
const captureRatePerMinute = ref(20)
|
||||
|
||||
let animationFrame: number | null = null
|
||||
|
||||
function pushHistory(value: number) {
|
||||
processingHistory.value = [...processingHistory.value, value].slice(-200)
|
||||
}
|
||||
|
||||
function animate() {
|
||||
const base = 140 + Math.sin(Date.now() / 900) * 40
|
||||
const jitter = (Math.random() - 0.5) * 30
|
||||
processingValue.value = Math.max(40, base + jitter)
|
||||
pushHistory(processingValue.value)
|
||||
|
||||
contextUpdatesPerMinute.value = Math.max(0, Math.min(60, contextUpdatesPerMinute.value + (Math.random() - 0.5) * 6))
|
||||
captureRatePerMinute.value = Math.max(0, Math.min(60, captureRatePerMinute.value + (Math.random() - 0.5) * 8))
|
||||
|
||||
animationFrame = requestAnimationFrame(animate)
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
for (let i = 0; i < 20; i += 1)
|
||||
pushHistory(120 + Math.random() * 50)
|
||||
animate()
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
if (animationFrame)
|
||||
cancelAnimationFrame(animationFrame)
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Story title="Processing Meter" group="gadgets">
|
||||
<Variant id="vision-processing" title="Vision Processing">
|
||||
<div :class="['p-4', 'max-w-3xl']">
|
||||
<ProcessingMeter
|
||||
title="Vision ticker"
|
||||
:processing-history="processingHistory"
|
||||
:processing-value="processingValue"
|
||||
processing-label="Inference latency"
|
||||
processing-unit="ms"
|
||||
:rate-value="Math.round(contextUpdatesPerMinute)"
|
||||
:rate-max="60"
|
||||
rate-label="Context updates"
|
||||
rate-unit="/min"
|
||||
:secondary-rate-value="Math.round(captureRatePerMinute)"
|
||||
:secondary-rate-max="60"
|
||||
secondary-rate-label="Capture rate"
|
||||
secondary-rate-unit="/min"
|
||||
/>
|
||||
</div>
|
||||
</Variant>
|
||||
</Story>
|
||||
</template>
|
||||
@@ -0,0 +1,116 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
|
||||
import LevelMeter from './level-meter.vue'
|
||||
import TimeSeriesChart from './time-series-chart.vue'
|
||||
|
||||
interface Props {
|
||||
title?: string
|
||||
processingHistory: Readonly<number[]>
|
||||
processingValue?: number | null
|
||||
processingUnit?: string
|
||||
processingLabel?: string
|
||||
processingMax?: number
|
||||
rateValue?: number
|
||||
rateMax?: number
|
||||
rateLabel?: string
|
||||
rateUnit?: string
|
||||
secondaryRateValue?: number
|
||||
secondaryRateMax?: number
|
||||
secondaryRateLabel?: string
|
||||
secondaryRateUnit?: string
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
title: 'Processing',
|
||||
processingValue: null,
|
||||
processingUnit: 'ms',
|
||||
processingLabel: 'Inference latency',
|
||||
processingMax: 0,
|
||||
rateValue: 0,
|
||||
rateMax: 60,
|
||||
rateLabel: 'Context updates',
|
||||
rateUnit: '/min',
|
||||
secondaryRateValue: undefined,
|
||||
secondaryRateMax: 60,
|
||||
secondaryRateLabel: 'Capture rate',
|
||||
secondaryRateUnit: '/min',
|
||||
})
|
||||
|
||||
const resolvedProcessingMax = computed(() => {
|
||||
const historyMax = props.processingHistory.length > 0
|
||||
? Math.max(...props.processingHistory)
|
||||
: 0
|
||||
return Math.max(1, props.processingMax || 0, historyMax)
|
||||
})
|
||||
|
||||
const processingValueResolved = computed(() => {
|
||||
if (props.processingValue !== null && props.processingValue !== undefined)
|
||||
return props.processingValue
|
||||
return props.processingHistory.at(-1) ?? 0
|
||||
})
|
||||
|
||||
const normalizedProcessingHistory = computed(() => {
|
||||
const maxValue = resolvedProcessingMax.value
|
||||
return props.processingHistory.map(value => Math.min(1, value / maxValue))
|
||||
})
|
||||
|
||||
const normalizedProcessingValue = computed(() => {
|
||||
return Math.min(1, processingValueResolved.value / resolvedProcessingMax.value)
|
||||
})
|
||||
|
||||
const formattedProcessingValue = computed(() => {
|
||||
return `${processingValueResolved.value.toFixed(0)}${props.processingUnit}`
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div :class="['flex', 'flex-col', 'gap-4', 'rounded-2xl', 'bg-white/70', 'p-4', 'shadow-sm', 'dark:bg-neutral-900/50']">
|
||||
<div :class="['flex', 'items-center', 'justify-between']">
|
||||
<div :class="['flex', 'flex-col', 'gap-1']">
|
||||
<div :class="['text-xs', 'uppercase', 'tracking-wide', 'text-neutral-400']">
|
||||
{{ title }}
|
||||
</div>
|
||||
<div :class="['text-base', 'font-semibold', 'text-neutral-700', 'dark:text-neutral-200']">
|
||||
{{ processingLabel }}
|
||||
</div>
|
||||
</div>
|
||||
<div :class="['text-sm', 'text-neutral-500', 'dark:text-neutral-400']">
|
||||
{{ formattedProcessingValue }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<TimeSeriesChart
|
||||
:is-active="false"
|
||||
:history="normalizedProcessingHistory"
|
||||
:current-value="normalizedProcessingValue"
|
||||
:show-legend="false"
|
||||
:show-header="false"
|
||||
:show-area="true"
|
||||
:show-active-indicator="false"
|
||||
:show-current-value="false"
|
||||
:height="72"
|
||||
:precision="0"
|
||||
:unit="processingUnit"
|
||||
/>
|
||||
|
||||
<div :class="['grid', 'gap-4', 'md:grid-cols-2']">
|
||||
<LevelMeter
|
||||
:level="rateValue"
|
||||
:min="0"
|
||||
:max="rateMax"
|
||||
:label="rateLabel"
|
||||
:unit="rateUnit"
|
||||
/>
|
||||
|
||||
<LevelMeter
|
||||
v-if="secondaryRateValue !== undefined"
|
||||
:level="secondaryRateValue"
|
||||
:min="0"
|
||||
:max="secondaryRateMax"
|
||||
:label="secondaryRateLabel"
|
||||
:unit="secondaryRateUnit"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -0,0 +1,45 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, onUnmounted, ref } from 'vue'
|
||||
|
||||
import TimelineRange from './timeline-range.vue'
|
||||
|
||||
const data = ref<number[]>([])
|
||||
const range = ref<[number, number] | null>(null)
|
||||
|
||||
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
function seedData() {
|
||||
const now = Date.now()
|
||||
data.value = Array.from({ length: 120 }, (_, index) => now - (120 - index) * 1000)
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
seedData()
|
||||
intervalId = setInterval(() => {
|
||||
data.value = [...data.value.slice(-200), Date.now()]
|
||||
}, 1000)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
if (intervalId)
|
||||
clearInterval(intervalId)
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Story title="Timeline Range" group="gadgets">
|
||||
<Variant id="basic" title="Basic">
|
||||
<div :class="['p-4', 'max-w-3xl']">
|
||||
<TimelineRange
|
||||
v-model:range="range"
|
||||
:data="data"
|
||||
:height="140"
|
||||
:bins="40"
|
||||
/>
|
||||
<div :class="['mt-3', 'text-xs', 'text-neutral-400']">
|
||||
{{ range ? `Range: ${new Date(range[0]).toLocaleTimeString()} - ${new Date(range[1]).toLocaleTimeString()}` : 'No range selected' }}
|
||||
</div>
|
||||
</div>
|
||||
</Variant>
|
||||
</Story>
|
||||
</template>
|
||||
@@ -0,0 +1,179 @@
|
||||
<script setup lang="ts">
|
||||
import { useElementBounding } from '@vueuse/core'
|
||||
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue'
|
||||
|
||||
import * as d3 from 'd3'
|
||||
|
||||
interface Props {
|
||||
data: number[]
|
||||
height?: number
|
||||
bins?: number
|
||||
range?: [number, number] | null
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
height: 120,
|
||||
bins: 40,
|
||||
range: null,
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
(event: 'update:range', value: [number, number] | null): void
|
||||
}>()
|
||||
|
||||
const containerRef = ref<HTMLDivElement | null>(null)
|
||||
const { width } = useElementBounding(containerRef, { windowResize: true })
|
||||
|
||||
let svg: d3.Selection<SVGSVGElement, unknown, null, undefined> | null = null
|
||||
let brushBehavior: d3.BrushBehavior<unknown> | null = null
|
||||
let xScale: d3.ScaleLinear<number, number> | null = null
|
||||
|
||||
const sanitizedData = computed(() => props.data.filter(value => Number.isFinite(value)))
|
||||
|
||||
function clearChart() {
|
||||
if (!containerRef.value)
|
||||
return
|
||||
containerRef.value.innerHTML = ''
|
||||
svg = null
|
||||
brushBehavior = null
|
||||
xScale = null
|
||||
}
|
||||
|
||||
function drawChart() {
|
||||
if (!containerRef.value)
|
||||
return
|
||||
|
||||
const containerWidth = Math.max(0, Math.floor(width.value || 0))
|
||||
if (containerWidth === 0)
|
||||
return
|
||||
|
||||
clearChart()
|
||||
|
||||
const margin = {
|
||||
top: 12,
|
||||
right: 14,
|
||||
bottom: 20,
|
||||
left: 28,
|
||||
}
|
||||
const chartWidth = Math.max(0, containerWidth - margin.left - margin.right)
|
||||
const chartHeight = Math.max(0, props.height - margin.top - margin.bottom)
|
||||
|
||||
const data = sanitizedData.value
|
||||
if (data.length === 0) {
|
||||
svg = d3.select(containerRef.value)
|
||||
.append('svg')
|
||||
.attr('width', containerWidth)
|
||||
.attr('height', props.height)
|
||||
return
|
||||
}
|
||||
|
||||
const [minValueRaw, maxValueRaw] = d3.extent(data) as [number, number]
|
||||
const minValue = Number.isFinite(minValueRaw) ? minValueRaw : 0
|
||||
const maxValue = Number.isFinite(maxValueRaw) ? maxValueRaw : minValue + 1
|
||||
const paddedMax = maxValue === minValue ? maxValue + 1 : maxValue
|
||||
|
||||
xScale = d3.scaleLinear()
|
||||
.domain([minValue, paddedMax])
|
||||
.range([0, chartWidth])
|
||||
|
||||
const bins = d3.bin()
|
||||
.domain(xScale.domain() as [number, number])
|
||||
.thresholds(props.bins)(data)
|
||||
|
||||
const yScale = d3.scaleLinear()
|
||||
.domain([0, d3.max(bins, bin => bin.length) || 1])
|
||||
.range([chartHeight, 0])
|
||||
|
||||
svg = d3.select(containerRef.value)
|
||||
.append('svg')
|
||||
.attr('width', containerWidth)
|
||||
.attr('height', props.height)
|
||||
.style('overflow', 'visible')
|
||||
|
||||
const chart = svg
|
||||
.append('g')
|
||||
.attr('transform', `translate(${margin.left},${margin.top})`)
|
||||
|
||||
chart
|
||||
.append('g')
|
||||
.attr('class', 'timeline-range-bars')
|
||||
.selectAll('rect')
|
||||
.data(bins)
|
||||
.join('rect')
|
||||
.attr('x', bin => xScale?.(bin.x0 ?? minValue) ?? 0)
|
||||
.attr('y', bin => yScale(bin.length))
|
||||
.attr('width', bin => Math.max(1, (xScale?.(bin.x1 ?? minValue) ?? 0) - (xScale?.(bin.x0 ?? minValue) ?? 0) - 1))
|
||||
.attr('height', bin => chartHeight - yScale(bin.length))
|
||||
.attr('rx', 2)
|
||||
.attr('fill', 'rgba(66,72,83,0.85)')
|
||||
|
||||
const axis = d3.axisBottom(xScale).ticks(4).tickFormat((d) => {
|
||||
if (typeof d === 'number')
|
||||
return new Date(d).toLocaleTimeString()
|
||||
return `${d}`
|
||||
})
|
||||
|
||||
chart
|
||||
.append('g')
|
||||
.attr('class', 'timeline-range-axis')
|
||||
.attr('transform', `translate(0,${chartHeight})`)
|
||||
.call(axis)
|
||||
.call(g => g.select('.domain').attr('opacity', 0.2))
|
||||
.call(g => g.selectAll('text').attr('fill', '#9CA1AE'))
|
||||
.call(g => g.selectAll('line').attr('opacity', 0.08))
|
||||
|
||||
brushBehavior = d3.brushX()
|
||||
.extent([[0, 0], [chartWidth, chartHeight]])
|
||||
.on('brush end', (event) => {
|
||||
if (!xScale)
|
||||
return
|
||||
if (!event.selection) {
|
||||
emit('update:range', null)
|
||||
return
|
||||
}
|
||||
const [start, end] = event.selection as [number, number]
|
||||
emit('update:range', [xScale.invert(start), xScale.invert(end)])
|
||||
})
|
||||
|
||||
const brushGroup = chart
|
||||
.append('g')
|
||||
.attr('class', 'timeline-range-brush')
|
||||
.call(brushBehavior)
|
||||
|
||||
brushGroup.selectAll('.selection')
|
||||
.attr('fill', 'rgba(114,163,183,0.25)')
|
||||
.attr('stroke', 'rgba(114,163,183,0.6)')
|
||||
|
||||
if (props.range) {
|
||||
const [start, end] = props.range
|
||||
brushGroup.call(brushBehavior.move, [xScale(start), xScale(end)])
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
drawChart()
|
||||
})
|
||||
|
||||
watch([width, sanitizedData, () => props.bins, () => props.height], () => {
|
||||
drawChart()
|
||||
})
|
||||
|
||||
watch(() => props.range, (nextRange) => {
|
||||
if (!svg || !brushBehavior || !xScale)
|
||||
return
|
||||
const brushGroup = svg.select<SVGGElement>('g.timeline-range-brush')
|
||||
if (!nextRange) {
|
||||
brushGroup.call(brushBehavior.move, null)
|
||||
return
|
||||
}
|
||||
brushGroup.call(brushBehavior.move, [xScale(nextRange[0]), xScale(nextRange[1])])
|
||||
})
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
clearChart()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div ref="containerRef" :class="['w-full']" />
|
||||
</template>
|
||||
@@ -10,4 +10,5 @@ export * from './use-chat-session/summary'
|
||||
export * from './use-optimistic'
|
||||
export * from './use-scroll-to-hash'
|
||||
export * from './use-versioned-local-storage'
|
||||
export * from './vision'
|
||||
export * from './whisper'
|
||||
|
||||
@@ -13,6 +13,7 @@ import { useMinecraftStore } from '../stores/modules/gaming-minecraft'
|
||||
import { useHearingStore } from '../stores/modules/hearing'
|
||||
import { useSpeechStore } from '../stores/modules/speech'
|
||||
import { useTwitterStore } from '../stores/modules/twitter'
|
||||
import { useVisionStore } from '../stores/modules/vision'
|
||||
|
||||
export interface Module {
|
||||
id: string
|
||||
@@ -33,6 +34,7 @@ export function useModulesList() {
|
||||
const consciousnessStore = useConsciousnessStore()
|
||||
const speechStore = useSpeechStore()
|
||||
const hearingStore = useHearingStore()
|
||||
const visionStore = useVisionStore()
|
||||
const discordStore = useDiscordStore()
|
||||
const twitterStore = useTwitterStore()
|
||||
const minecraftStore = useMinecraftStore()
|
||||
@@ -73,7 +75,7 @@ export function useModulesList() {
|
||||
description: t('settings.pages.modules.vision.description'),
|
||||
icon: 'i-solar:eye-closed-bold-duotone',
|
||||
to: '/settings/modules/vision',
|
||||
configured: false,
|
||||
configured: visionStore.configured,
|
||||
category: 'essential',
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './use-vision-inference'
|
||||
export * from './use-vision-workloads'
|
||||
@@ -0,0 +1,82 @@
|
||||
import type { ChatProvider } from '@xsai-ext/providers/utils'
|
||||
import type { CommonContentPart, Message } from '@xsai/shared-chat'
|
||||
|
||||
import type { VisionWorkloadId } from './use-vision-workloads'
|
||||
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { useLLM } from '../../stores/llm'
|
||||
import { useVisionStore } from '../../stores/modules/vision'
|
||||
import { useProvidersStore } from '../../stores/providers'
|
||||
import { getVisionWorkload } from './use-vision-workloads'
|
||||
|
||||
export interface VisionInferenceInput {
|
||||
imageDataUrl: string
|
||||
workloadId: VisionWorkloadId
|
||||
promptOverride?: string
|
||||
}
|
||||
|
||||
function parseDataUrl(dataUrl: string) {
|
||||
if (!dataUrl.startsWith('data:'))
|
||||
return { mimeType: 'image/png', base64: dataUrl, url: dataUrl }
|
||||
|
||||
const [, meta, data] = dataUrl.match(/^data:([^,]+),(.*)$/) || []
|
||||
const mimeType = meta?.split(';')[0] || 'image/png'
|
||||
const base64 = meta?.includes('base64') ? data : btoa(data)
|
||||
return {
|
||||
mimeType,
|
||||
base64,
|
||||
url: `data:${mimeType};base64,${base64}`,
|
||||
}
|
||||
}
|
||||
|
||||
export function useVisionInference() {
|
||||
const llmStore = useLLM()
|
||||
const providersStore = useProvidersStore()
|
||||
const visionStore = useVisionStore()
|
||||
const { activeProvider, activeModel } = storeToRefs(visionStore)
|
||||
|
||||
const lastText = ref('')
|
||||
|
||||
async function runVisionInference(input: VisionInferenceInput) {
|
||||
if (!activeProvider.value || !activeModel.value)
|
||||
throw new Error('Vision provider/model not configured')
|
||||
|
||||
const provider = await providersStore.getProviderInstance<ChatProvider>(activeProvider.value)
|
||||
const workload = getVisionWorkload(input.workloadId)
|
||||
const prompt = input.promptOverride ?? workload.prompt
|
||||
const { url } = parseDataUrl(input.imageDataUrl)
|
||||
|
||||
const contentParts: CommonContentPart[] = [
|
||||
{ type: 'text', text: prompt },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
const messages: Message[] = [
|
||||
{ role: 'user', content: contentParts },
|
||||
]
|
||||
|
||||
let buffer = ''
|
||||
await llmStore.stream(activeModel.value, provider, messages, {
|
||||
onStreamEvent: (event) => {
|
||||
if (event.type === 'text-delta') {
|
||||
buffer += event.text
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
lastText.value = buffer.trim()
|
||||
return lastText.value
|
||||
}
|
||||
|
||||
return {
|
||||
lastText,
|
||||
runVisionInference,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
export type VisionWorkloadId = 'screen:interpret' | 'screen:understand' | 'screen:ocr' | 'screen:ui-automation'
|
||||
|
||||
export interface VisionWorkloadConfig {
|
||||
id: VisionWorkloadId
|
||||
label: string
|
||||
description: string
|
||||
prompt: string
|
||||
}
|
||||
|
||||
export const VISION_WORKLOADS: VisionWorkloadConfig[] = [
|
||||
{
|
||||
id: 'screen:interpret',
|
||||
label: 'Screen interpret',
|
||||
description: 'Summarize what is on screen and relevant UI state.',
|
||||
prompt: [
|
||||
'You are an on-device vision assistant.',
|
||||
'Interpret the current screen in a concise, structured summary:',
|
||||
'- identify the active app or page',
|
||||
'- list key UI elements and their states',
|
||||
'- call out user intent or next likely action',
|
||||
'Keep it factual and short, avoid speculation.',
|
||||
].join('\n'),
|
||||
},
|
||||
{
|
||||
id: 'screen:understand',
|
||||
label: 'Screen understanding',
|
||||
description: 'Explain screen intent and key tasks.',
|
||||
prompt: [
|
||||
'Explain what the screen is for and what the user can do next.',
|
||||
'Focus on primary actions, warnings, and notable state changes.',
|
||||
].join('\n'),
|
||||
},
|
||||
{
|
||||
id: 'screen:ocr',
|
||||
label: 'OCR focus',
|
||||
description: 'Extract readable text from the screen.',
|
||||
prompt: [
|
||||
'Extract visible text from the screen.',
|
||||
'Return plain text, preserve structure with line breaks when possible.',
|
||||
].join('\n'),
|
||||
},
|
||||
{
|
||||
id: 'screen:ui-automation',
|
||||
label: 'UI automation',
|
||||
description: 'Describe actionable UI elements for automation.',
|
||||
prompt: [
|
||||
'Identify actionable UI elements (buttons, inputs, menus).',
|
||||
'Return a list of elements with labels and approximate purpose.',
|
||||
].join('\n'),
|
||||
},
|
||||
]
|
||||
|
||||
export function getVisionWorkload(id: VisionWorkloadId) {
|
||||
return VISION_WORKLOADS.find(workload => workload.id === id) || VISION_WORKLOADS[0]
|
||||
}
|
||||
@@ -6,8 +6,15 @@ import { ref, toRaw } from 'vue'
|
||||
|
||||
import { getEventSourceKey } from '../../utils/event-source'
|
||||
|
||||
export interface ContextHistoryEntry extends ContextMessage {
|
||||
sourceKey: string
|
||||
}
|
||||
|
||||
const CONTEXT_HISTORY_LIMIT = 400
|
||||
|
||||
export const useChatContextStore = defineStore('chat-context', () => {
|
||||
const activeContexts = ref<Record<string, ContextMessage[]>>({})
|
||||
const contextHistory = ref<ContextHistoryEntry[]>([])
|
||||
|
||||
function ingestContextMessage(envelope: ContextMessage) {
|
||||
const sourceKey = getEventSourceKey(envelope)
|
||||
@@ -21,10 +28,19 @@ export const useChatContextStore = defineStore('chat-context', () => {
|
||||
else if (envelope.strategy === ContextUpdateStrategy.AppendSelf) {
|
||||
activeContexts.value[sourceKey].push(envelope)
|
||||
}
|
||||
|
||||
contextHistory.value = [
|
||||
...contextHistory.value,
|
||||
{
|
||||
...envelope,
|
||||
sourceKey,
|
||||
},
|
||||
].slice(-CONTEXT_HISTORY_LIMIT)
|
||||
}
|
||||
|
||||
function resetContexts() {
|
||||
activeContexts.value = {}
|
||||
contextHistory.value = []
|
||||
}
|
||||
|
||||
function getContextsSnapshot() {
|
||||
@@ -35,5 +51,7 @@ export const useChatContextStore = defineStore('chat-context', () => {
|
||||
ingestContextMessage,
|
||||
resetContexts,
|
||||
getContextsSnapshot,
|
||||
activeContexts,
|
||||
contextHistory,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -6,3 +6,4 @@ export * from './gaming-minecraft'
|
||||
export * from './hearing'
|
||||
export * from './speech'
|
||||
export * from './twitter'
|
||||
export * from './vision'
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
export interface VisionAgentConfig {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
}
|
||||
|
||||
export const VISION_AGENTS: VisionAgentConfig[] = []
|
||||
@@ -0,0 +1,5 @@
|
||||
export * from './agents'
|
||||
export * from './orchestrator'
|
||||
export * from './processing-store'
|
||||
export * from './store'
|
||||
export * from './utils/model-catalog'
|
||||
@@ -0,0 +1,93 @@
|
||||
import type { CommonContentPart } from '@xsai/shared-chat'
|
||||
|
||||
import type { VisionWorkloadId } from '../../../composables/vision/use-vision-workloads'
|
||||
|
||||
import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
|
||||
import { defineStore, storeToRefs } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { useVisionInference } from '../../../composables/vision'
|
||||
import { getVisionWorkload } from '../../../composables/vision/use-vision-workloads'
|
||||
import { useModsServerChannelStore } from '../../mods/api/channel-server'
|
||||
import { useVisionStore } from './store'
|
||||
|
||||
export interface VisionCapturePayload {
|
||||
imageDataUrl: string
|
||||
workloadId: VisionWorkloadId
|
||||
sourceId?: string
|
||||
capturedAt?: number
|
||||
publishContext?: boolean
|
||||
}
|
||||
|
||||
export const useVisionOrchestratorStore = defineStore('vision-orchestrator', () => {
|
||||
const visionStore = useVisionStore()
|
||||
const { activeProvider, activeModel } = storeToRefs(visionStore)
|
||||
const modsServerChannelStore = useModsServerChannelStore()
|
||||
const { runVisionInference, lastText } = useVisionInference()
|
||||
|
||||
const lastResultText = ref('')
|
||||
const lastResultAt = ref<number | null>(null)
|
||||
const lastError = ref<string | null>(null)
|
||||
const lastWorkloadId = ref<VisionWorkloadId>('screen:interpret')
|
||||
|
||||
async function processCapture(payload: VisionCapturePayload) {
|
||||
if (!activeProvider.value || !activeModel.value)
|
||||
throw new Error('Vision model is not configured')
|
||||
|
||||
lastWorkloadId.value = payload.workloadId
|
||||
|
||||
const text = await runVisionInference({
|
||||
imageDataUrl: payload.imageDataUrl,
|
||||
workloadId: payload.workloadId,
|
||||
})
|
||||
|
||||
lastResultText.value = text
|
||||
lastResultAt.value = Date.now()
|
||||
lastError.value = null
|
||||
|
||||
if (payload.publishContext) {
|
||||
const workload = getVisionWorkload(payload.workloadId)
|
||||
const content: CommonContentPart[] = [
|
||||
{ type: 'text', text },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: payload.imageDataUrl,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
modsServerChannelStore.sendContextUpdate({
|
||||
strategy: ContextUpdateStrategy.ReplaceSelf,
|
||||
text,
|
||||
content,
|
||||
metadata: {
|
||||
module: 'vision',
|
||||
workload: workload.id,
|
||||
workloadLabel: workload.label,
|
||||
sourceId: payload.sourceId,
|
||||
capturedAt: payload.capturedAt,
|
||||
provider: activeProvider.value,
|
||||
model: activeModel.value,
|
||||
},
|
||||
})
|
||||
return { contextUpdates: 1, text }
|
||||
}
|
||||
|
||||
return { contextUpdates: 0, text }
|
||||
}
|
||||
|
||||
function recordError(error: unknown) {
|
||||
lastError.value = error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
|
||||
return {
|
||||
lastText,
|
||||
lastResultText,
|
||||
lastResultAt,
|
||||
lastError,
|
||||
lastWorkloadId,
|
||||
processCapture,
|
||||
recordError,
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,204 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { computed, ref, watch } from 'vue'
|
||||
|
||||
import { createResettableLocalStorage } from '../../../utils/resettable'
|
||||
|
||||
export interface VisionTickOutcome {
|
||||
capturedAt?: number
|
||||
contextUpdates?: number
|
||||
}
|
||||
|
||||
type VisionTickHandler = () => Promise<VisionTickOutcome | void> | VisionTickOutcome | void
|
||||
|
||||
const DEFAULT_CAPTURE_INTERVAL_MS = 3000
|
||||
const HISTORY_MAX_AGE_MS = 5 * 60 * 1000
|
||||
const PROCESSING_HISTORY_LIMIT = 240
|
||||
|
||||
function trimHistoryByAge(history: number[], maxAgeMs: number) {
|
||||
const cutoff = Date.now() - maxAgeMs
|
||||
while (history.length > 0 && history[0] < cutoff)
|
||||
history.shift()
|
||||
}
|
||||
|
||||
function countInWindow(history: number[], windowMs: number) {
|
||||
const cutoff = Date.now() - windowMs
|
||||
let count = 0
|
||||
for (let index = history.length - 1; index >= 0; index -= 1) {
|
||||
if (history[index] < cutoff)
|
||||
break
|
||||
count += 1
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
export const useVisionProcessingStore = defineStore('vision-processing', () => {
|
||||
const [captureIntervalMs, resetCaptureIntervalMs] = createResettableLocalStorage(
|
||||
'settings/vision/capture-interval-ms',
|
||||
DEFAULT_CAPTURE_INTERVAL_MS,
|
||||
)
|
||||
|
||||
const isRunning = ref(false)
|
||||
const isProcessing = ref(false)
|
||||
const tickCount = ref(0)
|
||||
const skippedTicks = ref(0)
|
||||
const captureCount = ref(0)
|
||||
const contextUpdateCount = ref(0)
|
||||
const lastTickAt = ref<number | null>(null)
|
||||
const lastCaptureAt = ref<number | null>(null)
|
||||
const lastContextUpdateAt = ref<number | null>(null)
|
||||
const lastProcessingDurationMs = ref<number | null>(null)
|
||||
const lastError = ref<string | null>(null)
|
||||
|
||||
const processingHistoryMs = ref<number[]>([])
|
||||
const captureHistory = ref<number[]>([])
|
||||
const contextUpdateHistory = ref<number[]>([])
|
||||
|
||||
let intervalHandle: ReturnType<typeof setInterval> | null = null
|
||||
const tickHandler = ref<VisionTickHandler | null>(null)
|
||||
|
||||
const captureRatePerMinute = computed(() => countInWindow(captureHistory.value, 60_000))
|
||||
const contextUpdateRatePerMinute = computed(() => countInWindow(contextUpdateHistory.value, 60_000))
|
||||
|
||||
const averageProcessingMs = computed(() => {
|
||||
if (processingHistoryMs.value.length === 0)
|
||||
return 0
|
||||
const total = processingHistoryMs.value.reduce((sum, value) => sum + value, 0)
|
||||
return total / processingHistoryMs.value.length
|
||||
})
|
||||
|
||||
function recordProcessingDuration(durationMs: number) {
|
||||
lastProcessingDurationMs.value = durationMs
|
||||
processingHistoryMs.value = [...processingHistoryMs.value, durationMs].slice(-PROCESSING_HISTORY_LIMIT)
|
||||
}
|
||||
|
||||
function recordCapture(capturedAt = Date.now()) {
|
||||
captureCount.value += 1
|
||||
lastCaptureAt.value = capturedAt
|
||||
captureHistory.value.push(capturedAt)
|
||||
trimHistoryByAge(captureHistory.value, HISTORY_MAX_AGE_MS)
|
||||
}
|
||||
|
||||
function recordContextUpdates(count = 1, updatedAt = Date.now()) {
|
||||
if (count <= 0)
|
||||
return
|
||||
|
||||
contextUpdateCount.value += count
|
||||
lastContextUpdateAt.value = updatedAt
|
||||
for (let index = 0; index < count; index += 1)
|
||||
contextUpdateHistory.value.push(updatedAt)
|
||||
trimHistoryByAge(contextUpdateHistory.value, HISTORY_MAX_AGE_MS)
|
||||
}
|
||||
|
||||
async function runTick() {
|
||||
if (!tickHandler.value)
|
||||
return
|
||||
if (isProcessing.value) {
|
||||
skippedTicks.value += 1
|
||||
return
|
||||
}
|
||||
|
||||
isProcessing.value = true
|
||||
lastTickAt.value = Date.now()
|
||||
tickCount.value += 1
|
||||
|
||||
const start = performance.now()
|
||||
|
||||
try {
|
||||
const outcome = await tickHandler.value()
|
||||
lastError.value = null
|
||||
|
||||
if (outcome?.capturedAt)
|
||||
recordCapture(outcome.capturedAt)
|
||||
if (outcome?.contextUpdates)
|
||||
recordContextUpdates(outcome.contextUpdates)
|
||||
}
|
||||
catch (error) {
|
||||
lastError.value = error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
finally {
|
||||
recordProcessingDuration(performance.now() - start)
|
||||
isProcessing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function startTicker(handler: VisionTickHandler) {
|
||||
tickHandler.value = handler
|
||||
if (isRunning.value)
|
||||
return
|
||||
|
||||
isRunning.value = true
|
||||
if (intervalHandle)
|
||||
clearInterval(intervalHandle)
|
||||
|
||||
void runTick()
|
||||
intervalHandle = setInterval(() => {
|
||||
void runTick()
|
||||
}, captureIntervalMs.value)
|
||||
}
|
||||
|
||||
function stopTicker() {
|
||||
isRunning.value = false
|
||||
if (intervalHandle)
|
||||
clearInterval(intervalHandle)
|
||||
intervalHandle = null
|
||||
}
|
||||
|
||||
function resetMetrics() {
|
||||
tickCount.value = 0
|
||||
skippedTicks.value = 0
|
||||
captureCount.value = 0
|
||||
contextUpdateCount.value = 0
|
||||
lastTickAt.value = null
|
||||
lastCaptureAt.value = null
|
||||
lastContextUpdateAt.value = null
|
||||
lastProcessingDurationMs.value = null
|
||||
lastError.value = null
|
||||
processingHistoryMs.value = []
|
||||
captureHistory.value = []
|
||||
contextUpdateHistory.value = []
|
||||
}
|
||||
|
||||
function resetState() {
|
||||
stopTicker()
|
||||
resetMetrics()
|
||||
resetCaptureIntervalMs()
|
||||
}
|
||||
|
||||
watch(captureIntervalMs, (next, previous) => {
|
||||
if (!isRunning.value)
|
||||
return
|
||||
if (next === previous)
|
||||
return
|
||||
|
||||
if (intervalHandle)
|
||||
clearInterval(intervalHandle)
|
||||
intervalHandle = setInterval(() => {
|
||||
void runTick()
|
||||
}, next)
|
||||
})
|
||||
|
||||
return {
|
||||
captureIntervalMs,
|
||||
isRunning,
|
||||
isProcessing,
|
||||
tickCount,
|
||||
skippedTicks,
|
||||
captureCount,
|
||||
contextUpdateCount,
|
||||
lastTickAt,
|
||||
lastCaptureAt,
|
||||
lastContextUpdateAt,
|
||||
lastProcessingDurationMs,
|
||||
lastError,
|
||||
processingHistoryMs,
|
||||
captureHistory,
|
||||
contextUpdateHistory,
|
||||
captureRatePerMinute,
|
||||
contextUpdateRatePerMinute,
|
||||
averageProcessingMs,
|
||||
startTicker,
|
||||
stopTicker,
|
||||
resetMetrics,
|
||||
resetState,
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,93 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { computed } from 'vue'
|
||||
|
||||
import { createResettableLocalStorage, createResettableRef } from '../../../utils/resettable'
|
||||
import { useProvidersStore } from '../../providers'
|
||||
|
||||
export const useVisionStore = defineStore('vision', () => {
|
||||
const providersStore = useProvidersStore()
|
||||
|
||||
const [activeProvider, resetActiveProvider] = createResettableLocalStorage('settings/vision/active-provider', '')
|
||||
const [activeModel, resetActiveModel] = createResettableLocalStorage('settings/vision/active-model', '')
|
||||
const [activeCustomModelName, resetActiveCustomModelName] = createResettableLocalStorage('settings/vision/active-custom-model', '')
|
||||
const [modelSearchQuery, resetModelSearchQuery] = createResettableRef('')
|
||||
|
||||
const providerMetadata = computed(() => {
|
||||
if (!activeProvider.value)
|
||||
return null
|
||||
|
||||
return providersStore.providerMetadata[activeProvider.value] ?? null
|
||||
})
|
||||
|
||||
const supportsModelListing = computed(() => {
|
||||
return providerMetadata.value?.capabilities.listModels !== undefined
|
||||
})
|
||||
|
||||
const providerModels = computed(() => {
|
||||
if (!activeProvider.value)
|
||||
return []
|
||||
|
||||
return providersStore.getModelsForProvider(activeProvider.value)
|
||||
})
|
||||
|
||||
const isLoadingActiveProviderModels = computed(() => {
|
||||
if (!activeProvider.value)
|
||||
return false
|
||||
|
||||
return providersStore.isLoadingModels[activeProvider.value] || false
|
||||
})
|
||||
|
||||
const activeProviderModelError = computed(() => {
|
||||
if (!activeProvider.value)
|
||||
return null
|
||||
|
||||
return providersStore.modelLoadError[activeProvider.value] || null
|
||||
})
|
||||
|
||||
const configured = computed(() => {
|
||||
return !!activeProvider.value && !!activeModel.value
|
||||
})
|
||||
|
||||
function resetModelSelection() {
|
||||
resetActiveModel()
|
||||
resetActiveCustomModelName()
|
||||
resetModelSearchQuery()
|
||||
}
|
||||
|
||||
async function loadModelsForProvider(provider: string) {
|
||||
if (provider && providerMetadata.value?.capabilities.listModels !== undefined) {
|
||||
await providersStore.fetchModelsForProvider(provider)
|
||||
}
|
||||
}
|
||||
|
||||
async function getModelsForProvider(provider: string) {
|
||||
if (provider && providerMetadata.value?.capabilities.listModels !== undefined) {
|
||||
return providersStore.getModelsForProvider(provider)
|
||||
}
|
||||
|
||||
return []
|
||||
}
|
||||
|
||||
function resetState() {
|
||||
resetActiveProvider()
|
||||
resetModelSelection()
|
||||
}
|
||||
|
||||
return {
|
||||
activeProvider,
|
||||
activeModel,
|
||||
customModelName: activeCustomModelName,
|
||||
modelSearchQuery,
|
||||
|
||||
supportsModelListing,
|
||||
providerModels,
|
||||
isLoadingActiveProviderModels,
|
||||
activeProviderModelError,
|
||||
configured,
|
||||
|
||||
resetModelSelection,
|
||||
loadModelsForProvider,
|
||||
getModelsForProvider,
|
||||
resetState,
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,55 @@
|
||||
export interface VisionModelInfo {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
tags: string[]
|
||||
recommendedFor: string[]
|
||||
deprecated?: boolean
|
||||
customizable?: boolean
|
||||
}
|
||||
|
||||
export const VISION_MODEL_CATALOG: VisionModelInfo[] = [
|
||||
{
|
||||
id: 'gpt-4o-mini-vision',
|
||||
name: 'GPT-4o mini (Vision)',
|
||||
description: 'Fast and cost-efficient for screen UI understanding and lightweight OCR.',
|
||||
tags: ['fast', 'ui', 'ocr'],
|
||||
recommendedFor: ['UI navigation', 'Quick captions', 'Low-latency feedback'],
|
||||
},
|
||||
{
|
||||
id: 'gpt-4o-vision',
|
||||
name: 'GPT-4o (Vision)',
|
||||
description: 'Stronger reasoning on complex layouts and multi-window scenes.',
|
||||
tags: ['accurate', 'ui', 'reasoning'],
|
||||
recommendedFor: ['Dense UIs', 'Multi-step analysis', 'Ambiguous screens'],
|
||||
},
|
||||
{
|
||||
id: 'claude-3.5-sonnet-vision',
|
||||
name: 'Claude 3.5 Sonnet (Vision)',
|
||||
description: 'Balanced quality for diagrams, docs, and structured UI reading.',
|
||||
tags: ['balanced', 'documents', 'charts'],
|
||||
recommendedFor: ['Docs and dashboards', 'Diagram reading', 'Summaries'],
|
||||
},
|
||||
{
|
||||
id: 'gemini-1.5-pro-vision',
|
||||
name: 'Gemini 1.5 Pro (Vision)',
|
||||
description: 'Great for longer-context screen stories and multi-step tasks.',
|
||||
tags: ['long-context', 'reasoning'],
|
||||
recommendedFor: ['Long sessions', 'Workflow tracking', 'Large screens'],
|
||||
},
|
||||
{
|
||||
id: 'llava-1.6',
|
||||
name: 'LLaVA 1.6 (Local)',
|
||||
description: 'Local-friendly baseline for offline or privacy-focused setups.',
|
||||
tags: ['local', 'offline'],
|
||||
recommendedFor: ['Offline mode', 'On-device evaluation', 'Privacy-first use'],
|
||||
},
|
||||
{
|
||||
id: 'custom',
|
||||
name: 'Custom Model',
|
||||
description: 'Use your own vision model ID from a provider or local runtime.',
|
||||
tags: ['custom'],
|
||||
recommendedFor: ['Self-hosted models', 'Experimental runtimes'],
|
||||
customizable: true,
|
||||
},
|
||||
]
|
||||
Reference in New Issue
Block a user