style: lint

plugins/airi-plugin-bilibili-laplace/tsdown.config.ts

@@ -2,9 +2,9 @@ import { defineConfig } from 'tsdown'
 
 export default defineConfig([
   {
-    entry: ['./src/index.ts'],
     dts: true,
-    unused: true,
+    entry: ['./src/index.ts'],
     publint: true,
+    unused: true,
   },
 ])

plugins/airi-plugin-claude-code/src/cli.ts

@@ -16,11 +16,11 @@ import { resolveComma, toArray } from './utils/general'
 interface Options {
   config?: string
   configLoader?: 'auto' | 'native' | 'unconfig'
-  noConfig?: boolean
   debug?: boolean | string | string[]
-  logLevel?: LogLevelString.Log | LogLevelString.Warning | LogLevelString.Error
-  failOnWarn?: boolean
   env?: Record<string, string>
+  failOnWarn?: boolean
+  logLevel?: LogLevelString.Error | LogLevelString.Log | LogLevelString.Warning
+  noConfig?: boolean
   quiet?: boolean
 }
 
@@ -30,7 +30,7 @@ const cli = cac('airi-plugin-claude-code-cli')
 cli.help().version(version)
 
 cli
-  .command('send', 'Pass Claude Code hook event to Channel Server', { ignoreOptionDefaultValue: true, allowUnknownOptions: true })
+  .command('send', 'Pass Claude Code hook event to Channel Server', { allowUnknownOptions: true, ignoreOptionDefaultValue: true })
   .option('-c, --config <filename>', 'Use a custom config file')
   .option('--config-loader <loader>', 'Config loader to use: auto, native, unconfig', { default: 'auto' })
   .option('--no-config', 'Disable config file')
@@ -68,10 +68,10 @@ cli
     const hookEvent = JSON.parse(stdinInput) as HookInput
 
     if (hookEvent.hook_event_name === 'UserPromptSubmit') {
-      const channelServer = new Client({ name: 'proj-airi:plugin-claude-code', autoConnect: false })
+      const channelServer = new Client({ autoConnect: false, name: 'proj-airi:plugin-claude-code' })
       await channelServer.connect()
 
-      channelServer.send({ type: 'input:text', data: { text: hookEvent.prompt } })
+      channelServer.send({ data: { text: hookEvent.prompt }, type: 'input:text' })
     }
   })
 

plugins/airi-plugin-claude-code/src/utils/general.ts

@@ -1,39 +1,3 @@
-/**
- * https://github.com/rolldown/tsdown/blob/a7e267ab7f4e836e836dab5cecf029fc35fd1939/src/utils/general.ts
- */
-export function toArray<T>(
-  val: T | T[] | null | undefined,
-  defaultValue?: T,
-): T[] {
-  if (Array.isArray(val)) {
-    return val
-  }
-  else if (val == null) {
-    if (defaultValue)
-      return [defaultValue]
-    return []
-  }
-  else {
-    return [val]
-  }
-}
-
-export function resolveComma<T extends string>(arr: T[]): T[] {
-  return arr.flatMap(format => format.split(',') as T[])
-}
-
-export function resolveRegex<T>(str: T): T | RegExp {
-  if (
-    typeof str === 'string'
-    && str.length > 2
-    && str[0] === '/'
-    && str.at(-1) === '/'
-  ) {
-    return new RegExp(str.slice(1, -1))
-  }
-  return str
-}
-
 export function debounce<T extends (...args: any[]) => any>(
   fn: T,
   wait: number,
@@ -49,15 +13,51 @@ export function debounce<T extends (...args: any[]) => any>(
   } as T
 }
 
+export function resolveComma<T extends string>(arr: T[]): T[] {
+  return arr.flatMap(format => format.split(',') as T[])
+}
+
+export function resolveRegex<T>(str: T): RegExp | T {
+  if (
+    typeof str === 'string'
+    && str.length > 2
+    && str[0] === '/'
+    && str.at(-1) === '/'
+  ) {
+    return new RegExp(str.slice(1, -1))
+  }
+  return str
+}
+
 export function slash(string: string): string {
   return string.replaceAll('\\', '/')
 }
 
+/**
+ * https://github.com/rolldown/tsdown/blob/a7e267ab7f4e836e836dab5cecf029fc35fd1939/src/utils/general.ts
+ */
+export function toArray<T>(
+  val: null | T | T[] | undefined,
+  defaultValue?: T,
+): T[] {
+  if (Array.isArray(val)) {
+    return val
+  }
+  else if (val == null) {
+    if (defaultValue)
+      return [defaultValue]
+    return []
+  }
+  else {
+    return [val]
+  }
+}
+
 export const noop = <T>(v: T): T => v
 
 export function matchPattern(
   id: string,
-  patterns: (string | RegExp)[],
+  patterns: (RegExp | string)[],
 ): boolean {
   return patterns.some((pattern) => {
     if (pattern instanceof RegExp) {

plugins/airi-plugin-claude-code/tsdown.config.ts

@@ -2,11 +2,11 @@ import { defineConfig } from 'tsdown'
 
 export default defineConfig([
   {
+    dts: true,
     entry: ['./src/run.ts'],
     inlineOnly: [],
     platform: 'node',
-    dts: true,
-    unused: true,
     publint: true,
+    unused: true,
   },
 ])

plugins/airi-plugin-homeassistant/tsdown.config.ts

@@ -2,9 +2,9 @@ import { defineConfig } from 'tsdown'
 
 export default defineConfig([
   {
-    entry: ['./src/index.ts'],
     dts: true,
-    unused: true,
+    entry: ['./src/index.ts'],
     publint: true,
+    unused: true,
   },
 ])

plugins/airi-plugin-web-extension/entrypoints/background.ts

@@ -35,30 +35,10 @@ let lastStatusSentAt = 0
 let connectionKey = ''
 let eventaContext: ReturnType<typeof createRuntimeEventaContext>['context'] | undefined
 
-async function refreshClient() {
-  const nextKey = `${settings.enabled}:${settings.wsUrl}:${settings.token}`
-  if (nextKey !== connectionKey) {
-    connectionKey = nextKey
-    if (state.client)
-      state.client.close()
-    state.client = null
-    state.connected = false
-  }
-  await ensureClient(state, settings)
-}
-
-function buildNotifyKey(payload: { url: string, title?: string, videoId?: string }) {
+function buildNotifyKey(payload: { title?: string, url: string, videoId?: string }) {
   return [payload.videoId, payload.title, payload.url].filter(Boolean).join('|')
 }
 
-function shouldNotifyVideo(payload: { url: string, title?: string, videoId?: string }) {
-  const key = buildNotifyKey(payload)
-  if (!key || key === lastVideoNotifyKey)
-    return false
-  lastVideoNotifyKey = key
-  return true
-}
-
 function emitStatus() {
   const now = Date.now()
   if (now - lastStatusSentAt < 300)
@@ -68,18 +48,6 @@ function emitStatus() {
   eventaContext?.emit(backgroundStatusChanged, toStatus(state, settings))
 }
 
-async function updateSettings(partial: Partial<ExtensionSettings>) {
-  settings = await saveSettings(partial)
-  await refreshClient()
-  emitStatus()
-}
-
-async function init() {
-  settings = await loadSettings()
-  await refreshClient()
-  emitStatus()
-}
-
 function handleContentMessage(message: ContentToBackgroundMessage) {
   switch (message.type) {
     case 'content:page': {
@@ -91,15 +59,6 @@ function handleContentMessage(message: ContentToBackgroundMessage) {
       emitStatus()
       break
     }
-    case 'content:video': {
-      const payload = {
-        ...message.payload,
-        site: message.payload.site === 'unknown' ? detectSiteFromUrl(message.payload.url) : message.payload.site,
-      }
-      handleVideoContext(state, settings, payload, { notify: shouldNotifyVideo(payload) })
-      emitStatus()
-      break
-    }
     case 'content:subtitle': {
       const payload = {
         ...message.payload,
@@ -109,6 +68,15 @@ function handleContentMessage(message: ContentToBackgroundMessage) {
       emitStatus()
       break
     }
+    case 'content:video': {
+      const payload = {
+        ...message.payload,
+        site: message.payload.site === 'unknown' ? detectSiteFromUrl(message.payload.url) : message.payload.site,
+      }
+      handleVideoContext(state, settings, payload, { notify: shouldNotifyVideo(payload) })
+      emitStatus()
+      break
+    }
     case 'content:vision:frame': {
       state.lastVisionFrameAt = Date.now()
       emitStatus()
@@ -117,6 +85,38 @@ function handleContentMessage(message: ContentToBackgroundMessage) {
   }
 }
 
+async function init() {
+  settings = await loadSettings()
+  await refreshClient()
+  emitStatus()
+}
+
+async function refreshClient() {
+  const nextKey = `${settings.enabled}:${settings.wsUrl}:${settings.token}`
+  if (nextKey !== connectionKey) {
+    connectionKey = nextKey
+    if (state.client)
+      state.client.close()
+    state.client = null
+    state.connected = false
+  }
+  await ensureClient(state, settings)
+}
+
+function shouldNotifyVideo(payload: { title?: string, url: string, videoId?: string }) {
+  const key = buildNotifyKey(payload)
+  if (!key || key === lastVideoNotifyKey)
+    return false
+  lastVideoNotifyKey = key
+  return true
+}
+
+async function updateSettings(partial: Partial<ExtensionSettings>) {
+  settings = await saveSettings(partial)
+  await refreshClient()
+  emitStatus()
+}
+
 export default defineBackground(() => {
   const { context } = createRuntimeEventaContext()
   eventaContext = context

plugins/airi-plugin-web-extension/entrypoints/content.ts

@@ -1,11 +1,11 @@
 import { startContentObserver } from '../src/content'
 
 export default defineContentScript({
+  main() {
+    startContentObserver()
+  },
   matches: [
     '*://*/*',
   ],
   runAt: 'document_idle',
-  main() {
-    startContentObserver()
-  },
 })

plugins/airi-plugin-web-extension/entrypoints/popup/stores/popup.ts

@@ -13,14 +13,14 @@ export const usePopupStore = createGlobalState(() => {
   const initialized = ref(false)
 
   const form = reactive<ExtensionSettings>({
-    wsUrl: '',
-    token: '',
     enabled: true,
-    sendPageContext: true,
-    sendVideoContext: true,
-    sendSubtitles: true,
-    sendSparkNotify: true,
     enableVision: false,
+    sendPageContext: true,
+    sendSparkNotify: true,
+    sendSubtitles: true,
+    sendVideoContext: true,
+    token: '',
+    wsUrl: '',
   })
 
   const connected = computed(() => status.value?.connected ?? false)
@@ -111,18 +111,18 @@ export const usePopupStore = createGlobalState(() => {
   }
 
   return {
-    status,
-    syncing,
-    form,
-    connected,
-    lastVideo,
-    lastSubtitle,
-    lastError,
-    init,
-    refresh,
     applySettings,
-    toggle,
     captureFrame,
     clearLastError,
+    connected,
+    form,
+    init,
+    lastError,
+    lastSubtitle,
+    lastVideo,
+    refresh,
+    status,
+    syncing,
+    toggle,
   }
 })

plugins/airi-plugin-web-extension/src/background/client.ts

@@ -16,8 +16,8 @@ export interface ClientState {
   connected: boolean
   lastError?: string
   lastPage?: PageContextPayload
-  lastVideo?: VideoContextPayload
   lastSubtitle?: SubtitlePayload
+  lastVideo?: VideoContextPayload
   lastVisionFrameAt?: number
 }
 
@@ -28,30 +28,13 @@ export function createClientState(): ClientState {
   }
 }
 
-function createIdentity() {
-  return {
-    kind: 'plugin',
-    plugin: {
-      id: PLUGIN_NAME,
-      version: typeof packageJSON.version === 'string' ? packageJSON.version : undefined,
-    },
-    id: nanoid(),
-    labels: {
-      runtime: 'web-extension',
-    },
-  }
-}
+export function disconnectClient(state: ClientState) {
+  if (!state.client)
+    return
 
-export function toStatus(state: ClientState, settings: ExtensionSettings): ExtensionStatus {
-  return {
-    connected: state.connected,
-    lastError: state.lastError,
-    settings,
-    lastPage: state.lastPage,
-    lastVideo: state.lastVideo,
-    lastSubtitle: state.lastSubtitle,
-    lastVisionFrameAt: state.lastVisionFrameAt,
-  }
+  state.client.close()
+  state.client = null
+  state.connected = false
 }
 
 export async function ensureClient(state: ClientState, settings: ExtensionSettings) {
@@ -65,20 +48,20 @@ export async function ensureClient(state: ClientState, settings: ExtensionSettin
   }
 
   const client = new Client({
-    name: PLUGIN_NAME,
-    url: settings.wsUrl,
-    token: settings.token || undefined,
-    identity: createIdentity(),
-    possibleEvents: ['context:update', 'spark:notify', 'spark:emit'],
     autoConnect: false,
     autoReconnect: true,
+    identity: createIdentity(),
+    name: PLUGIN_NAME,
+    onClose: () => {
+      state.connected = false
+    },
     onError: (error) => {
       state.connected = false
       state.lastError = errorMessageFromValue(error)
     },
-    onClose: () => {
-      state.connected = false
-    },
+    possibleEvents: ['context:update', 'spark:notify', 'spark:emit'],
+    token: settings.token || undefined,
+    url: settings.wsUrl,
   })
 
   state.client = client
@@ -94,49 +77,6 @@ export async function ensureClient(state: ClientState, settings: ExtensionSettin
   }
 }
 
-export function disconnectClient(state: ClientState) {
-  if (!state.client)
-    return
-
-  state.client.close()
-  state.client = null
-  state.connected = false
-}
-
-function sendContextUpdate(state: ClientState, update: Omit<ContextUpdate, 'id' | 'contextId'> & Partial<Pick<ContextUpdate, 'id' | 'contextId'>>) {
-  if (!state.client || !state.connected)
-    return
-
-  const id = update.id ?? nanoid()
-  state.client.send({
-    type: 'context:update',
-    data: {
-      id,
-      contextId: update.contextId ?? id,
-      ...update,
-    },
-  })
-}
-
-function sendSparkNotify(state: ClientState, data: { headline: string, note?: string, payload?: Record<string, unknown> }) {
-  if (!state.client || !state.connected)
-    return
-
-  state.client.send({
-    type: 'spark:notify',
-    data: {
-      id: nanoid(),
-      eventId: nanoid(),
-      kind: 'ping',
-      urgency: 'soon',
-      headline: data.headline,
-      note: data.note,
-      payload: data.payload,
-      destinations: ['character'],
-    },
-  })
-}
-
 export function handlePageContext(state: ClientState, settings: ExtensionSettings, payload: PageContextPayload) {
   state.lastPage = payload
 
@@ -144,17 +84,41 @@ export function handlePageContext(state: ClientState, settings: ExtensionSetting
     return
 
   sendContextUpdate(state, {
-    strategy: ContextUpdateStrategy.ReplaceSelf,
     lane: 'web:page',
-    text: `User is browsing: ${payload.title} (${payload.url}).`,
     metadata: {
-      source: 'web-extension',
-      site: payload.site,
-      url: payload.url,
-      title: payload.title,
       description: payload.description,
       language: payload.language,
+      site: payload.site,
+      source: 'web-extension',
+      title: payload.title,
+      url: payload.url,
     },
+    strategy: ContextUpdateStrategy.ReplaceSelf,
+    text: `User is browsing: ${payload.title} (${payload.url}).`,
+  })
+}
+
+export function handleSubtitle(state: ClientState, settings: ExtensionSettings, payload: SubtitlePayload) {
+  state.lastSubtitle = payload
+
+  if (!settings.enabled || !settings.sendSubtitles)
+    return
+
+  sendContextUpdate(state, {
+    lane: 'web:subtitle',
+    metadata: {
+      endMs: payload.endMs,
+      isAuto: payload.isAuto,
+      language: payload.language,
+      site: payload.site,
+      source: 'web-extension',
+      startMs: payload.startMs,
+      title: payload.title,
+      url: payload.url,
+      videoId: payload.videoId,
+    },
+    strategy: ContextUpdateStrategy.ReplaceSelf,
+    text: `Subtitle: ${payload.text}`,
   })
 }
 
@@ -178,22 +142,36 @@ export function handleVideoContext(
       headline,
       note: payload.channel ? `Channel: ${payload.channel}` : undefined,
       payload: {
-        site: payload.site,
-        url: payload.url,
-        title: payload.title,
         channel: payload.channel,
-        videoId: payload.videoId,
-        durationSec: payload.durationSec,
         currentTimeSec: payload.currentTimeSec,
-        isPlaying: payload.isPlaying,
+        durationSec: payload.durationSec,
         isLive: payload.isLive,
+        isPlaying: payload.isPlaying,
+        site: payload.site,
+        title: payload.title,
+        url: payload.url,
+        videoId: payload.videoId,
       },
     })
   }
 
   sendContextUpdate(state, {
-    strategy: ContextUpdateStrategy.ReplaceSelf,
     lane: 'web:video',
+    metadata: {
+      channel: payload.channel,
+      currentTimeSec: payload.currentTimeSec,
+      durationSec: payload.durationSec,
+      isLive: payload.isLive,
+      isPlaying: payload.isPlaying,
+      playbackRate: payload.playbackRate,
+      playerSize: payload.playerSize,
+      site: payload.site,
+      source: 'web-extension',
+      title: payload.title,
+      url: payload.url,
+      videoId: payload.videoId,
+    },
+    strategy: ContextUpdateStrategy.ReplaceSelf,
     text: [
       headline,
       payload.channel ? `Channel: ${payload.channel}.` : undefined,
@@ -202,43 +180,65 @@ export function handleVideoContext(
         : undefined,
       payload.url ? `URL: ${payload.url}.` : undefined,
     ].filter(Boolean).join(' '),
-    metadata: {
-      source: 'web-extension',
-      site: payload.site,
-      url: payload.url,
-      title: payload.title,
-      channel: payload.channel,
-      videoId: payload.videoId,
-      durationSec: payload.durationSec,
-      currentTimeSec: payload.currentTimeSec,
-      isPlaying: payload.isPlaying,
-      playbackRate: payload.playbackRate,
-      isLive: payload.isLive,
-      playerSize: payload.playerSize,
-    },
   })
 }
 
-export function handleSubtitle(state: ClientState, settings: ExtensionSettings, payload: SubtitlePayload) {
-  state.lastSubtitle = payload
+export function toStatus(state: ClientState, settings: ExtensionSettings): ExtensionStatus {
+  return {
+    connected: state.connected,
+    lastError: state.lastError,
+    lastPage: state.lastPage,
+    lastSubtitle: state.lastSubtitle,
+    lastVideo: state.lastVideo,
+    lastVisionFrameAt: state.lastVisionFrameAt,
+    settings,
+  }
+}
 
-  if (!settings.enabled || !settings.sendSubtitles)
+function createIdentity() {
+  return {
+    id: nanoid(),
+    kind: 'plugin',
+    labels: {
+      runtime: 'web-extension',
+    },
+    plugin: {
+      id: PLUGIN_NAME,
+      version: typeof packageJSON.version === 'string' ? packageJSON.version : undefined,
+    },
+  }
+}
+
+function sendContextUpdate(state: ClientState, update: Omit<ContextUpdate, 'contextId' | 'id'> & Partial<Pick<ContextUpdate, 'contextId' | 'id'>>) {
+  if (!state.client || !state.connected)
     return
 
-  sendContextUpdate(state, {
-    strategy: ContextUpdateStrategy.ReplaceSelf,
-    lane: 'web:subtitle',
-    text: `Subtitle: ${payload.text}`,
-    metadata: {
-      source: 'web-extension',
-      site: payload.site,
-      url: payload.url,
-      title: payload.title,
-      videoId: payload.videoId,
-      language: payload.language,
-      startMs: payload.startMs,
-      endMs: payload.endMs,
-      isAuto: payload.isAuto,
+  const id = update.id ?? nanoid()
+  state.client.send({
+    data: {
+      contextId: update.contextId ?? id,
+      id,
+      ...update,
     },
+    type: 'context:update',
+  })
+}
+
+function sendSparkNotify(state: ClientState, data: { headline: string, note?: string, payload?: Record<string, unknown> }) {
+  if (!state.client || !state.connected)
+    return
+
+  state.client.send({
+    data: {
+      destinations: ['character'],
+      eventId: nanoid(),
+      headline: data.headline,
+      id: nanoid(),
+      kind: 'ping',
+      note: data.note,
+      payload: data.payload,
+      urgency: 'soon',
+    },
+    type: 'spark:notify',
   })
 }

plugins/airi-plugin-web-extension/src/content/index.ts

@@ -8,14 +8,26 @@ const SUBTITLE_DEDUPE_WINDOW = 2000
 
 const lastPayloadByType = new Map<string, string>()
 
-function safeSend(message: ContentToBackgroundMessage) {
-  const serialized = JSON.stringify(message.payload)
-  const lastSerialized = lastPayloadByType.get(message.type)
-  if (serialized === lastSerialized)
-    return
+export function startContentObserver() {
+  const site = detectSiteFromUrl(location.href)
+  safeSend({ payload: buildPageContext(site), type: 'content:page' })
+  const stopVideo = observeVideo(site)
 
-  lastPayloadByType.set(message.type, serialized)
-  void browser.runtime.sendMessage(message).catch(() => {})
+  browser.runtime.onMessage.addListener((message: BackgroundToContentMessage) => {
+    if (message.type === 'background:request-vision-frame') {
+      const video = document.querySelector('video') as HTMLVideoElement | null
+      if (!video)
+        return
+
+      const frame = captureVisionFrame(site, video)
+      if (frame)
+        safeSend({ payload: frame, type: 'content:vision:frame' })
+    }
+  })
+
+  return () => {
+    stopVideo?.()
+  }
 }
 
 function buildPageContext(site: VideoSite): PageContextPayload {
@@ -23,11 +35,11 @@ function buildPageContext(site: VideoSite): PageContextPayload {
   const ogDescription = normalizeText(document.querySelector('meta[property="og:description"]')?.getAttribute('content'))
 
   return {
-    site,
-    url: location.href,
-    title: normalizeText(document.title),
     description: description || ogDescription || undefined,
     language: document.documentElement.lang || undefined,
+    site,
+    title: normalizeText(document.title),
+    url: location.href,
   }
 }
 
@@ -41,39 +53,52 @@ function buildVideoContext(site: VideoSite, video: HTMLVideoElement, includeProg
   const rect = video.getBoundingClientRect()
 
   return {
-    site,
-    url,
-    title: title || normalizeText(document.title),
     channel: channel || undefined,
-    videoId,
-    durationSec,
     currentTimeSec,
-    isPlaying: !video.paused && !video.ended,
+    durationSec,
     isMuted: video.muted,
-    volume: Number.isFinite(video.volume) ? Number(video.volume.toFixed(2)) : undefined,
+    isPlaying: !video.paused && !video.ended,
     playbackRate: Number.isFinite(video.playbackRate) ? Number(video.playbackRate.toFixed(2)) : undefined,
-    playerSize: rect.width && rect.height ? { width: Math.round(rect.width), height: Math.round(rect.height) } : undefined,
+    playerSize: rect.width && rect.height ? { height: Math.round(rect.height), width: Math.round(rect.width) } : undefined,
+    site,
+    title: title || normalizeText(document.title),
+    url,
+    videoId,
+    volume: Number.isFinite(video.volume) ? Number(video.volume.toFixed(2)) : undefined,
   }
 }
 
-function findVideoTitle(site: VideoSite) {
-  if (site === 'youtube') {
-    return (
-      document.querySelector('ytd-watch-metadata h1 yt-formatted-string')?.textContent
-      || document.querySelector('h1.title yt-formatted-string')?.textContent
-      || document.querySelector('h1.title')?.textContent
-    )
-  }
+function captureVisionFrame(site: VideoSite, video: HTMLVideoElement): null | VisionFramePayload {
+  const canvas = document.createElement('canvas')
+  const width = Math.min(480, Math.max(1, Math.floor(video.videoWidth)))
+  const height = Math.min(270, Math.max(1, Math.floor(video.videoHeight)))
 
-  if (site === 'bilibili') {
-    return (
-      document.querySelector('h1.video-title')?.textContent
-      || document.querySelector('.video-title')?.textContent
-      || document.querySelector('h1')?.textContent
-    )
-  }
+  if (!width || !height)
+    return null
 
-  return document.querySelector('h1')?.textContent
+  canvas.width = width
+  canvas.height = height
+
+  const ctx = canvas.getContext('2d')
+  if (!ctx)
+    return null
+
+  try {
+    ctx.drawImage(video, 0, 0, width, height)
+    return {
+      capturedAt: Date.now(),
+      dataUrl: canvas.toDataURL('image/jpeg', 0.6),
+      height,
+      site,
+      title: normalizeText(findVideoTitle(site)) || undefined,
+      url: location.href,
+      videoId: extractVideoId(site, location.href),
+      width,
+    }
+  }
+  catch {
+    return null
+  }
 }
 
 function findChannelName(site: VideoSite) {
@@ -96,54 +121,24 @@ function findChannelName(site: VideoSite) {
   return undefined
 }
 
-function observeTextTracks(site: VideoSite, video: HTMLVideoElement, onSubtitle: (payload: SubtitlePayload) => void) {
-  const seen = new Map<string, number>()
-
-  const handleCueChange = (track: TextTrack) => {
-    const cues = Array.from(track.activeCues ?? []) as TextTrackCue[]
-    for (const cue of cues) {
-      const text = normalizeText((cue as VTTCue).text ?? '')
-      if (!text)
-        continue
-
-      const key = `${text}:${Math.floor(cue.startTime * 1000)}`
-      const now = Date.now()
-      const lastSeen = seen.get(key)
-      if (lastSeen && now - lastSeen < SUBTITLE_DEDUPE_WINDOW)
-        continue
-
-      seen.set(key, now)
-      onSubtitle({
-        site,
-        url: location.href,
-        title: normalizeText(findVideoTitle(site)) || undefined,
-        videoId: extractVideoId(site, location.href),
-        text,
-        language: (track.language || track.label || undefined),
-        startMs: Math.floor(cue.startTime * 1000),
-        endMs: Math.floor(cue.endTime * 1000),
-      })
-    }
+function findVideoTitle(site: VideoSite) {
+  if (site === 'youtube') {
+    return (
+      document.querySelector('ytd-watch-metadata h1 yt-formatted-string')?.textContent
+      || document.querySelector('h1.title yt-formatted-string')?.textContent
+      || document.querySelector('h1.title')?.textContent
+    )
   }
 
-  const attach = () => {
-    const tracks = Array.from(video.textTracks ?? [])
-    for (const track of tracks) {
-      if (track.kind && !['subtitles', 'captions'].includes(track.kind))
-        continue
-
-      if (track.mode === 'disabled')
-        track.mode = 'hidden'
-      track.oncuechange = () => handleCueChange(track)
-    }
+  if (site === 'bilibili') {
+    return (
+      document.querySelector('h1.video-title')?.textContent
+      || document.querySelector('.video-title')?.textContent
+      || document.querySelector('h1')?.textContent
+    )
   }
 
-  attach()
-
-  const observer = new MutationObserver(() => attach())
-  observer.observe(video, { attributes: true, childList: true, subtree: true })
-
-  return () => observer.disconnect()
+  return document.querySelector('h1')?.textContent
 }
 
 function observeSubtitleDom(site: VideoSite, onSubtitle: (payload: SubtitlePayload) => void) {
@@ -167,10 +162,10 @@ function observeSubtitleDom(site: VideoSite, onSubtitle: (payload: SubtitlePaylo
     lastText = text
     onSubtitle({
       site,
-      url: location.href,
-      title: normalizeText(findVideoTitle(site)) || undefined,
-      videoId: extractVideoId(site, location.href),
       text,
+      title: normalizeText(findVideoTitle(site)) || undefined,
+      url: location.href,
+      videoId: extractVideoId(site, location.href),
     })
   }
 
@@ -185,37 +180,54 @@ function observeSubtitleDom(site: VideoSite, onSubtitle: (payload: SubtitlePaylo
   }
 }
 
-function captureVisionFrame(site: VideoSite, video: HTMLVideoElement): VisionFramePayload | null {
-  const canvas = document.createElement('canvas')
-  const width = Math.min(480, Math.max(1, Math.floor(video.videoWidth)))
-  const height = Math.min(270, Math.max(1, Math.floor(video.videoHeight)))
+function observeTextTracks(site: VideoSite, video: HTMLVideoElement, onSubtitle: (payload: SubtitlePayload) => void) {
+  const seen = new Map<string, number>()
 
-  if (!width || !height)
-    return null
+  const handleCueChange = (track: TextTrack) => {
+    const cues = Array.from(track.activeCues ?? []) as TextTrackCue[]
+    for (const cue of cues) {
+      const text = normalizeText((cue as VTTCue).text ?? '')
+      if (!text)
+        continue
 
-  canvas.width = width
-  canvas.height = height
+      const key = `${text}:${Math.floor(cue.startTime * 1000)}`
+      const now = Date.now()
+      const lastSeen = seen.get(key)
+      if (lastSeen && now - lastSeen < SUBTITLE_DEDUPE_WINDOW)
+        continue
 
-  const ctx = canvas.getContext('2d')
-  if (!ctx)
-    return null
-
-  try {
-    ctx.drawImage(video, 0, 0, width, height)
-    return {
-      site,
-      url: location.href,
-      videoId: extractVideoId(site, location.href),
-      title: normalizeText(findVideoTitle(site)) || undefined,
-      capturedAt: Date.now(),
-      width,
-      height,
-      dataUrl: canvas.toDataURL('image/jpeg', 0.6),
+      seen.set(key, now)
+      onSubtitle({
+        endMs: Math.floor(cue.endTime * 1000),
+        language: (track.language || track.label || undefined),
+        site,
+        startMs: Math.floor(cue.startTime * 1000),
+        text,
+        title: normalizeText(findVideoTitle(site)) || undefined,
+        url: location.href,
+        videoId: extractVideoId(site, location.href),
+      })
     }
   }
-  catch {
-    return null
+
+  const attach = () => {
+    const tracks = Array.from(video.textTracks ?? [])
+    for (const track of tracks) {
+      if (track.kind && !['captions', 'subtitles'].includes(track.kind))
+        continue
+
+      if (track.mode === 'disabled')
+        track.mode = 'hidden'
+      track.oncuechange = () => handleCueChange(track)
+    }
   }
+
+  attach()
+
+  const observer = new MutationObserver(() => attach())
+  observer.observe(video, { attributes: true, childList: true, subtree: true })
+
+  return () => observer.disconnect()
 }
 
 function observeVideo(site: VideoSite) {
@@ -228,11 +240,11 @@ function observeVideo(site: VideoSite) {
     if (!video)
       return
 
-    safeSend({ type: 'content:video', payload: buildVideoContext(site, video, includeProgress) })
+    safeSend({ payload: buildVideoContext(site, video, includeProgress), type: 'content:video' })
   }
 
   const sendPage = () => {
-    safeSend({ type: 'content:page', payload: buildPageContext(site) })
+    safeSend({ payload: buildPageContext(site), type: 'content:page' })
   }
 
   const onPlayback = () => sendVideo(true)
@@ -253,8 +265,8 @@ function observeVideo(site: VideoSite) {
     stopTracks?.()
     stopDomSubtitles?.()
 
-    stopTracks = observeTextTracks(site, video, payload => safeSend({ type: 'content:subtitle', payload }))
-    stopDomSubtitles = observeSubtitleDom(site, payload => safeSend({ type: 'content:subtitle', payload }))
+    stopTracks = observeTextTracks(site, video, payload => safeSend({ payload, type: 'content:subtitle' }))
+    stopDomSubtitles = observeSubtitleDom(site, payload => safeSend({ payload, type: 'content:subtitle' }))
 
     sendPage()
     sendVideo(false)
@@ -315,24 +327,12 @@ function observeVideo(site: VideoSite) {
   }
 }
 
-export function startContentObserver() {
-  const site = detectSiteFromUrl(location.href)
-  safeSend({ type: 'content:page', payload: buildPageContext(site) })
-  const stopVideo = observeVideo(site)
+function safeSend(message: ContentToBackgroundMessage) {
+  const serialized = JSON.stringify(message.payload)
+  const lastSerialized = lastPayloadByType.get(message.type)
+  if (serialized === lastSerialized)
+    return
 
-  browser.runtime.onMessage.addListener((message: BackgroundToContentMessage) => {
-    if (message.type === 'background:request-vision-frame') {
-      const video = document.querySelector('video') as HTMLVideoElement | null
-      if (!video)
-        return
-
-      const frame = captureVisionFrame(site, video)
-      if (frame)
-        safeSend({ type: 'content:vision:frame', payload: frame })
-    }
-  })
-
-  return () => {
-    stopVideo?.()
-  }
+  lastPayloadByType.set(message.type, serialized)
+  void browser.runtime.sendMessage(message).catch(() => {})
 }

plugins/airi-plugin-web-extension/src/shared/constants.ts

@@ -3,14 +3,14 @@ import type { ExtensionSettings } from './types'
 export const DEFAULT_WS_URL = 'ws://localhost:6121/ws'
 
 export const DEFAULT_SETTINGS: ExtensionSettings = {
-  wsUrl: DEFAULT_WS_URL,
-  token: '',
   enabled: true,
-  sendPageContext: true,
-  sendVideoContext: true,
-  sendSubtitles: true,
-  sendSparkNotify: true,
   enableVision: false,
+  sendPageContext: true,
+  sendSparkNotify: true,
+  sendSubtitles: true,
+  sendVideoContext: true,
+  token: '',
+  wsUrl: DEFAULT_WS_URL,
 }
 
 export const STORAGE_KEY = 'airi:web-extension:settings'

plugins/airi-plugin-web-extension/src/shared/eventa-runtime.ts

@@ -9,9 +9,9 @@ const runtimeInstanceId = nanoid()
 interface EventaRuntimeMessage {
   __eventa: true
   channel: string
+  detail?: unknown
   sourceId: string
   type?: string
-  detail?: unknown
 }
 
 type RuntimeEventListener = (event: Event) => void
@@ -35,31 +35,31 @@ class RuntimeEventTarget implements EventTarget {
     this.listeners.get(type)?.set(listener, handler)
   }
 
-  removeEventListener(type: string, listener: EventListenerOrEventListenerObject | null) {
-    if (!listener)
-      return
-
-    this.listeners.get(type)?.delete(listener)
-  }
-
   dispatchEvent(event: Event) {
     const detail = 'detail' in event ? (event as CustomEvent).detail : undefined
     this.send({
       __eventa: true,
       channel: EVENTA_RUNTIME_CHANNEL,
+      detail,
       sourceId: runtimeInstanceId,
       type: event.type,
-      detail,
     })
 
     return true
   }
 
   emit(type: string, detail?: unknown) {
-    const event = { type, detail } as CustomEvent
+    const event = { detail, type } as CustomEvent
     for (const listener of this.listeners.get(type)?.values() ?? [])
       listener(event)
   }
+
+  removeEventListener(type: string, listener: EventListenerOrEventListenerObject | null) {
+    if (!listener)
+      return
+
+    this.listeners.get(type)?.delete(listener)
+  }
 }
 
 export function createRuntimeEventaContext() {
@@ -68,8 +68,8 @@ export function createRuntimeEventaContext() {
   })
 
   const { context, dispose } = createContext(eventTarget, {
-    messageEventName: EVENTA_MESSAGE_EVENT,
     errorEventName: false,
+    messageEventName: EVENTA_MESSAGE_EVENT,
   })
 
   const runtimeListener = (message: unknown) => {

plugins/airi-plugin-web-extension/src/shared/sites.ts

@@ -38,6 +38,6 @@ export function extractVideoId(site: VideoSite, url: string): string | undefined
   return undefined
 }
 
-export function normalizeText(value: string | null | undefined) {
+export function normalizeText(value: null | string | undefined) {
   return value?.replace(/\s+/g, ' ').trim() || ''
 }

plugins/airi-plugin-web-extension/src/shared/types.ts

@@ -1,78 +1,78 @@
-export type VideoSite = 'youtube' | 'bilibili' | 'unknown'
-
-export interface PageContextPayload {
-  site: VideoSite
-  url: string
-  title: string
-  description?: string
-  language?: string
-}
-
-export interface VideoContextPayload {
-  site: VideoSite
-  url: string
-  title: string
-  channel?: string
-  videoId?: string
-  durationSec?: number
-  currentTimeSec?: number
-  isPlaying?: boolean
-  isMuted?: boolean
-  volume?: number
-  playbackRate?: number
-  isLive?: boolean
-  playerSize?: { width: number, height: number }
-}
-
-export interface SubtitlePayload {
-  site: VideoSite
-  url: string
-  videoId?: string
-  title?: string
-  text: string
-  language?: string
-  startMs?: number
-  endMs?: number
-  isAuto?: boolean
-}
-
-export interface VisionFramePayload {
-  site: VideoSite
-  url: string
-  videoId?: string
-  title?: string
-  capturedAt: number
-  width: number
-  height: number
-  dataUrl: string
-}
+export type BackgroundToContentMessage
+  = | { type: 'background:request-vision-frame' }
 
 export type ContentToBackgroundMessage
-  = | { type: 'content:page', payload: PageContextPayload }
-    | { type: 'content:video', payload: VideoContextPayload }
-    | { type: 'content:subtitle', payload: SubtitlePayload }
-    | { type: 'content:vision:frame', payload: VisionFramePayload }
+  = | { payload: PageContextPayload, type: 'content:page' }
+    | { payload: SubtitlePayload, type: 'content:subtitle' }
+    | { payload: VideoContextPayload, type: 'content:video' }
+    | { payload: VisionFramePayload, type: 'content:vision:frame' }
 
 export interface ExtensionSettings {
-  wsUrl: string
-  token: string
   enabled: boolean
-  sendPageContext: boolean
-  sendVideoContext: boolean
-  sendSubtitles: boolean
-  sendSparkNotify: boolean
   enableVision: boolean
+  sendPageContext: boolean
+  sendSparkNotify: boolean
+  sendSubtitles: boolean
+  sendVideoContext: boolean
+  token: string
+  wsUrl: string
 }
 
 export interface ExtensionStatus {
   connected: boolean
   lastError?: string
-  settings: ExtensionSettings
   lastPage?: PageContextPayload
-  lastVideo?: VideoContextPayload
   lastSubtitle?: SubtitlePayload
+  lastVideo?: VideoContextPayload
   lastVisionFrameAt?: number
+  settings: ExtensionSettings
 }
 
-export type BackgroundToContentMessage
-  = | { type: 'background:request-vision-frame' }
+export interface PageContextPayload {
+  description?: string
+  language?: string
+  site: VideoSite
+  title: string
+  url: string
+}
+
+export interface SubtitlePayload {
+  endMs?: number
+  isAuto?: boolean
+  language?: string
+  site: VideoSite
+  startMs?: number
+  text: string
+  title?: string
+  url: string
+  videoId?: string
+}
+
+export interface VideoContextPayload {
+  channel?: string
+  currentTimeSec?: number
+  durationSec?: number
+  isLive?: boolean
+  isMuted?: boolean
+  isPlaying?: boolean
+  playbackRate?: number
+  playerSize?: { height: number, width: number }
+  site: VideoSite
+  title: string
+  url: string
+  videoId?: string
+  volume?: number
+}
+
+export type VideoSite = 'bilibili' | 'unknown' | 'youtube'
+
+export interface VisionFramePayload {
+  capturedAt: number
+  dataUrl: string
+  height: number
+  site: VideoSite
+  title?: string
+  url: string
+  videoId?: string
+  width: number
+}

plugins/airi-plugin-web-extension/wxt.config.ts

@@ -8,18 +8,18 @@ type VitePlugin = NonNullable<WxtViteConfig['plugins']>[number]
 
 // See https://wxt.dev/api/config.html
 export default defineConfig({
-  modules: ['@wxt-dev/module-vue'],
   manifest: {
-    name: 'AIRI Web Extension',
-    description: 'Capture web context (videos, pages, subtitles) for Project AIRI.',
-    permissions: ['storage', 'tabs'],
-    optional_host_permissions: [
-      '*://*/*',
-    ],
     action: {
       default_title: 'AIRI Web Extension',
     },
+    description: 'Capture web context (videos, pages, subtitles) for Project AIRI.',
+    name: 'AIRI Web Extension',
+    optional_host_permissions: [
+      '*://*/*',
+    ],
+    permissions: ['storage', 'tabs'],
   },
+  modules: ['@wxt-dev/module-vue'],
   vite: () => {
     return {
       plugins: [