feat(stage-ui): should handle WebGPU device-loss (#1680)
--------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
@@ -51,7 +51,7 @@
|
|||||||
},
|
},
|
||||||
"inlinedDependencies": {
|
"inlinedDependencies": {
|
||||||
"@electron-toolkit/preload": "3.0.2",
|
"@electron-toolkit/preload": "3.0.2",
|
||||||
"@moeru/eventa": "catalog:",
|
"@moeru/eventa": "1.0.0-beta.4",
|
||||||
"async-mutex": "0.5.0",
|
"async-mutex": "0.5.0",
|
||||||
"nanoid": [
|
"nanoid": [
|
||||||
"5.1.6",
|
"5.1.6",
|
||||||
|
|||||||
@@ -15,10 +15,13 @@ vi.mock('../../../composables/use-inference-status', () => ({
|
|||||||
removeInferenceStatus: vi.fn(),
|
removeInferenceStatus: vi.fn(),
|
||||||
}))
|
}))
|
||||||
|
|
||||||
|
const recordDeviceLoss = vi.fn()
|
||||||
vi.mock('../coordinator', () => ({
|
vi.mock('../coordinator', () => ({
|
||||||
getGPUCoordinator: () => ({
|
getGPUCoordinator: () => ({
|
||||||
requestAllocation: vi.fn(() => ({ modelId: 'test', estimatedBytes: 0 })),
|
requestAllocation: vi.fn(() => ({ modelId: 'test', estimatedBytes: 0 })),
|
||||||
release: vi.fn(),
|
release: vi.fn(),
|
||||||
|
touch: vi.fn(),
|
||||||
|
recordDeviceLoss,
|
||||||
}),
|
}),
|
||||||
getLoadQueue: () => ({
|
getLoadQueue: () => ({
|
||||||
enqueue: vi.fn((_id: string, _p: number, loader: () => Promise<unknown>) => loader()),
|
enqueue: vi.fn((_id: string, _p: number, loader: () => Promise<unknown>) => loader()),
|
||||||
@@ -76,3 +79,33 @@ describe('classifyError phase integration', () => {
|
|||||||
expect(classifyError(new Error('tensor shape mismatch'), 'inference')).toBe('INFERENCE_FAILED')
|
expect(classifyError(new Error('tensor shape mismatch'), 'inference')).toBe('INFERENCE_FAILED')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('kokoro adapter - device loss resilience', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers()
|
||||||
|
recordDeviceLoss.mockClear()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers()
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should start with zero device-loss count and null manifest', async () => {
|
||||||
|
const { createKokoroAdapter } = await import('./kokoro')
|
||||||
|
const adapter = createKokoroAdapter()
|
||||||
|
|
||||||
|
expect(adapter.deviceLossCount).toBe(0)
|
||||||
|
expect(adapter.manifest).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should expose manifest and deviceLossCount as readonly getters', async () => {
|
||||||
|
const { createKokoroAdapter } = await import('./kokoro')
|
||||||
|
const adapter = createKokoroAdapter()
|
||||||
|
|
||||||
|
// Getters should be defined and callable
|
||||||
|
expect(typeof adapter.deviceLossCount).toBe('number')
|
||||||
|
// Manifest is explicitly null before any load
|
||||||
|
expect(adapter.manifest).toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|||||||
@@ -13,10 +13,10 @@ import { defaultPerfTracer } from '@proj-airi/stage-shared'
|
|||||||
import { Mutex } from 'async-mutex'
|
import { Mutex } from 'async-mutex'
|
||||||
|
|
||||||
import { removeInferenceStatus, updateInferenceStatus } from '../../../composables/use-inference-status'
|
import { removeInferenceStatus, updateInferenceStatus } from '../../../composables/use-inference-status'
|
||||||
import { MAX_RESTARTS, MODEL_NAMES, RESTART_DELAY_MS, TIMEOUTS } from '../constants'
|
import { DEVICE_LOSS_WASM_THRESHOLD, MAX_RESTARTS, MODEL_NAMES, RESTART_DELAY_MS, TIMEOUTS } from '../constants'
|
||||||
import { getGPUCoordinator, getLoadQueue, MODEL_VRAM_ESTIMATES } from '../coordinator'
|
import { getGPUCoordinator, getLoadQueue, MODEL_VRAM_ESTIMATES } from '../coordinator'
|
||||||
import { LOAD_PRIORITY } from '../load-queue'
|
import { LOAD_PRIORITY } from '../load-queue'
|
||||||
import { classifyError, createRequestId } from '../protocol'
|
import { classifyDeviceLossReason, classifyError, createRequestId } from '../protocol'
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Types
|
// Types
|
||||||
@@ -41,6 +41,16 @@ export interface KokoroAdapter {
|
|||||||
|
|
||||||
/** Current state */
|
/** Current state */
|
||||||
readonly state: 'idle' | 'loading' | 'ready' | 'running' | 'error' | 'terminated'
|
readonly state: 'idle' | 'loading' | 'ready' | 'running' | 'error' | 'terminated'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Snapshot of the last successful load config, or null if never loaded.
|
||||||
|
* `device` reflects the device actually used (post WASM promotion / worker
|
||||||
|
* fallback), which may differ from the device requested by the caller.
|
||||||
|
*/
|
||||||
|
readonly manifest: { quantization: string, device: string } | null
|
||||||
|
|
||||||
|
/** Number of WebGPU device-loss events observed by this adapter */
|
||||||
|
readonly deviceLossCount: number
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -153,6 +163,11 @@ function waitForWorkerMessage<T = any>(
|
|||||||
// Factory
|
// Factory
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
interface KokoroManifest {
|
||||||
|
quantization: string
|
||||||
|
device: string
|
||||||
|
}
|
||||||
|
|
||||||
export function createKokoroAdapter(): KokoroAdapter {
|
export function createKokoroAdapter(): KokoroAdapter {
|
||||||
let worker: Worker | null = null
|
let worker: Worker | null = null
|
||||||
let state: KokoroAdapter['state'] = 'idle'
|
let state: KokoroAdapter['state'] = 'idle'
|
||||||
@@ -162,6 +177,13 @@ export function createKokoroAdapter(): KokoroAdapter {
|
|||||||
let currentModelStatusId: string | null = null
|
let currentModelStatusId: string | null = null
|
||||||
let errorListener: ((event: ErrorEvent) => void) | null = null
|
let errorListener: ((event: ErrorEvent) => void) | null = null
|
||||||
|
|
||||||
|
// NOTICE: Device-loss resilience state. `lastManifest` records the last
|
||||||
|
// successful load config so scheduleRestart can reconstruct context if the
|
||||||
|
// worker died. `deviceLossCount` tracks WebGPU device-loss events so we
|
||||||
|
// can promote to WASM after repeated failures (see DEVICE_LOSS_WASM_THRESHOLD).
|
||||||
|
let lastManifest: KokoroManifest | null = null
|
||||||
|
let deviceLossCount = 0
|
||||||
|
|
||||||
const operationMutex = new Mutex()
|
const operationMutex = new Mutex()
|
||||||
const lifecycleMutex = new Mutex()
|
const lifecycleMutex = new Mutex()
|
||||||
|
|
||||||
@@ -174,9 +196,22 @@ export function createKokoroAdapter(): KokoroAdapter {
|
|||||||
worker.addEventListener('error', errorListener)
|
worker.addEventListener('error', errorListener)
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleWorkerError(_event: ErrorEvent | Error): void {
|
function handleWorkerError(event: ErrorEvent | Error): void {
|
||||||
state = 'error'
|
state = 'error'
|
||||||
operationMutex.cancel()
|
operationMutex.cancel()
|
||||||
|
|
||||||
|
// Record device-loss telemetry before teardown so the coordinator sees it
|
||||||
|
// even if the adapter is never used again.
|
||||||
|
const code = classifyError(event instanceof Error ? event : (event as ErrorEvent).error ?? event)
|
||||||
|
if (code === 'DEVICE_LOST') {
|
||||||
|
deviceLossCount++
|
||||||
|
getGPUCoordinator().recordDeviceLoss({
|
||||||
|
modelId: currentModelStatusId ?? MODEL_NAMES.KOKORO,
|
||||||
|
reason: classifyDeviceLossReason(event instanceof Error ? event : (event as ErrorEvent).error ?? event),
|
||||||
|
occurredAt: Date.now(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
destroyWorker()
|
destroyWorker()
|
||||||
scheduleRestart()
|
scheduleRestart()
|
||||||
}
|
}
|
||||||
@@ -237,6 +272,22 @@ export function createKokoroAdapter(): KokoroAdapter {
|
|||||||
device: string,
|
device: string,
|
||||||
options?: { onProgress?: (p: ProgressPayload) => void },
|
options?: { onProgress?: (p: ProgressPayload) => void },
|
||||||
): Promise<Voices> {
|
): Promise<Voices> {
|
||||||
|
// NOTICE: Proactive WASM promotion. If this adapter has suffered repeated
|
||||||
|
// WebGPU device-loss events, webgpu is unreliable on this device and we
|
||||||
|
// should not keep retrying. The worker's per-load dtype/device fallback
|
||||||
|
// chain handles transient failures; this guard handles persistent ones.
|
||||||
|
let effectiveDevice = device
|
||||||
|
if (
|
||||||
|
device === 'webgpu'
|
||||||
|
&& deviceLossCount >= DEVICE_LOSS_WASM_THRESHOLD
|
||||||
|
) {
|
||||||
|
console.warn(
|
||||||
|
`[KokoroAdapter] ${deviceLossCount} device-loss events recorded, `
|
||||||
|
+ `promoting load from webgpu to wasm.`,
|
||||||
|
)
|
||||||
|
effectiveDevice = 'wasm'
|
||||||
|
}
|
||||||
|
|
||||||
await ensureStarted()
|
await ensureStarted()
|
||||||
|
|
||||||
return defaultPerfTracer.withMeasure('inference', 'kokoro-load-model', () => operationMutex.runExclusive(async () => {
|
return defaultPerfTracer.withMeasure('inference', 'kokoro-load-model', () => operationMutex.runExclusive(async () => {
|
||||||
@@ -248,7 +299,7 @@ export function createKokoroAdapter(): KokoroAdapter {
|
|||||||
removeInferenceStatus(currentModelStatusId)
|
removeInferenceStatus(currentModelStatusId)
|
||||||
currentModelStatusId = modelStatusId
|
currentModelStatusId = modelStatusId
|
||||||
|
|
||||||
updateInferenceStatus(modelStatusId, { state: 'downloading', device: device as any })
|
updateInferenceStatus(modelStatusId, { state: 'downloading', device: effectiveDevice as any })
|
||||||
|
|
||||||
// Use the global load queue to serialize model loads across all adapters
|
// Use the global load queue to serialize model loads across all adapters
|
||||||
return getLoadQueue().enqueue(modelStatusId, LOAD_PRIORITY.TTS, async () => {
|
return getLoadQueue().enqueue(modelStatusId, LOAD_PRIORITY.TTS, async () => {
|
||||||
@@ -275,7 +326,7 @@ export function createKokoroAdapter(): KokoroAdapter {
|
|||||||
type: 'load-model',
|
type: 'load-model',
|
||||||
requestId,
|
requestId,
|
||||||
modelId: MODEL_NAMES.KOKORO,
|
modelId: MODEL_NAMES.KOKORO,
|
||||||
device,
|
device: effectiveDevice,
|
||||||
dtype: quantization,
|
dtype: quantization,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -290,14 +341,18 @@ export function createKokoroAdapter(): KokoroAdapter {
|
|||||||
const estimated = MODEL_VRAM_ESTIMATES[estimateKey] ?? 165 * 1024 * 1024
|
const estimated = MODEL_VRAM_ESTIMATES[estimateKey] ?? 165 * 1024 * 1024
|
||||||
allocationToken = coordinator.requestAllocation(`kokoro-${quantization}`, estimated)
|
allocationToken = coordinator.requestAllocation(`kokoro-${quantization}`, estimated)
|
||||||
|
|
||||||
|
// Record manifest so consumers can inspect how the adapter resolved
|
||||||
|
// device selection after fallback / WASM promotion.
|
||||||
|
lastManifest = { quantization, device: (response.device ?? effectiveDevice) as string }
|
||||||
|
|
||||||
state = 'ready'
|
state = 'ready'
|
||||||
updateInferenceStatus(modelStatusId, { state: 'ready', device: (response.device ?? device) as any })
|
updateInferenceStatus(modelStatusId, { state: 'ready', device: (response.device ?? effectiveDevice) as any })
|
||||||
onSuccess()
|
onSuccess()
|
||||||
if (!voices)
|
if (!voices)
|
||||||
throw new Error('Kokoro worker did not return voice metadata')
|
throw new Error('Kokoro worker did not return voice metadata')
|
||||||
return voices
|
return voices
|
||||||
})
|
})
|
||||||
}), { quantization, device }).catch((error) => {
|
}), { quantization, device: effectiveDevice }).catch((error) => {
|
||||||
handleWorkerError(error instanceof Error ? error : new Error(String(error)))
|
handleWorkerError(error instanceof Error ? error : new Error(String(error)))
|
||||||
throw error
|
throw error
|
||||||
})
|
})
|
||||||
@@ -364,6 +419,8 @@ export function createKokoroAdapter(): KokoroAdapter {
|
|||||||
getVoices,
|
getVoices,
|
||||||
terminate: terminateAdapter,
|
terminate: terminateAdapter,
|
||||||
get state() { return state },
|
get state() { return state },
|
||||||
|
get manifest() { return lastManifest },
|
||||||
|
get deviceLossCount() { return deviceLossCount },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,10 +13,10 @@ import { defaultPerfTracer } from '@proj-airi/stage-shared'
|
|||||||
import { Mutex } from 'async-mutex'
|
import { Mutex } from 'async-mutex'
|
||||||
|
|
||||||
import { removeInferenceStatus, updateInferenceStatus } from '../../../composables/use-inference-status'
|
import { removeInferenceStatus, updateInferenceStatus } from '../../../composables/use-inference-status'
|
||||||
import { MAX_RESTARTS, MODEL_NAMES, RESTART_DELAY_MS, TIMEOUTS } from '../constants'
|
import { DEVICE_LOSS_WASM_THRESHOLD, MAX_RESTARTS, MODEL_NAMES, RESTART_DELAY_MS, TIMEOUTS } from '../constants'
|
||||||
import { getGPUCoordinator, getLoadQueue, MODEL_VRAM_ESTIMATES } from '../coordinator'
|
import { getGPUCoordinator, getLoadQueue, MODEL_VRAM_ESTIMATES } from '../coordinator'
|
||||||
import { LOAD_PRIORITY } from '../load-queue'
|
import { LOAD_PRIORITY } from '../load-queue'
|
||||||
import { createRequestId } from '../protocol'
|
import { classifyDeviceLossReason, classifyError, createRequestId } from '../protocol'
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Types
|
// Types
|
||||||
@@ -64,6 +64,15 @@ export interface WhisperAdapter {
|
|||||||
* Returns an unsubscribe function.
|
* Returns an unsubscribe function.
|
||||||
*/
|
*/
|
||||||
onMessage: (handler: (event: WhisperEvent) => void) => () => void
|
onMessage: (handler: (event: WhisperEvent) => void) => () => void
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Snapshot of the last successful load, or null if never loaded.
|
||||||
|
* `device` reflects what the worker actually used (post-fallback).
|
||||||
|
*/
|
||||||
|
readonly manifest: { device: string } | null
|
||||||
|
|
||||||
|
/** Number of WebGPU device-loss events observed by this adapter */
|
||||||
|
readonly deviceLossCount: number
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -86,11 +95,26 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
|
|||||||
let errorListener: ((event: ErrorEvent) => void) | null = null
|
let errorListener: ((event: ErrorEvent) => void) | null = null
|
||||||
const messageHandlers = new Set<(event: WhisperEvent) => void>()
|
const messageHandlers = new Set<(event: WhisperEvent) => void>()
|
||||||
|
|
||||||
|
// NOTICE: Device-loss resilience state. See kokoro.ts for rationale.
|
||||||
|
let lastManifest: { device: string } | null = null
|
||||||
|
let deviceLossCount = 0
|
||||||
|
|
||||||
const operationMutex = new Mutex()
|
const operationMutex = new Mutex()
|
||||||
|
|
||||||
function handleWorkerError(_event: ErrorEvent | Error): void {
|
function handleWorkerError(event: ErrorEvent | Error): void {
|
||||||
state = 'error'
|
state = 'error'
|
||||||
operationMutex.cancel()
|
operationMutex.cancel()
|
||||||
|
|
||||||
|
const code = classifyError(event instanceof Error ? event : (event as ErrorEvent).error ?? event)
|
||||||
|
if (code === 'DEVICE_LOST') {
|
||||||
|
deviceLossCount++
|
||||||
|
getGPUCoordinator().recordDeviceLoss({
|
||||||
|
modelId: MODEL_NAMES.WHISPER,
|
||||||
|
reason: classifyDeviceLossReason(event instanceof Error ? event : (event as ErrorEvent).error ?? event),
|
||||||
|
occurredAt: Date.now(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
destroyWorker()
|
destroyWorker()
|
||||||
scheduleRestart()
|
scheduleRestart()
|
||||||
}
|
}
|
||||||
@@ -208,9 +232,20 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
|
|||||||
async function load(
|
async function load(
|
||||||
onProgress?: (p: ProgressPayload) => void,
|
onProgress?: (p: ProgressPayload) => void,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
// NOTICE: Proactive WASM promotion after repeated device-loss events.
|
||||||
|
// See kokoro.ts for rationale. Whisper always requests 'webgpu' from the
|
||||||
|
// caller today, so we only check the promotion threshold.
|
||||||
|
const requestedDevice = deviceLossCount >= DEVICE_LOSS_WASM_THRESHOLD ? 'wasm' : 'webgpu'
|
||||||
|
if (requestedDevice === 'wasm') {
|
||||||
|
console.warn(
|
||||||
|
`[WhisperAdapter] ${deviceLossCount} device-loss events recorded, `
|
||||||
|
+ `promoting load from webgpu to wasm.`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
return operationMutex.runExclusive(async () => {
|
return operationMutex.runExclusive(async () => {
|
||||||
state = 'loading'
|
state = 'loading'
|
||||||
updateInferenceStatus(MODEL_NAMES.WHISPER, { state: 'downloading', device: 'webgpu' })
|
updateInferenceStatus(MODEL_NAMES.WHISPER, { state: 'downloading', device: requestedDevice as any })
|
||||||
|
|
||||||
return getLoadQueue().enqueue(MODEL_NAMES.WHISPER, LOAD_PRIORITY.ASR, async () => {
|
return getLoadQueue().enqueue(MODEL_NAMES.WHISPER, LOAD_PRIORITY.ASR, async () => {
|
||||||
const w = ensureWorker()
|
const w = ensureWorker()
|
||||||
@@ -230,7 +265,7 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
w.postMessage({ type: 'load-model', requestId, modelId: MODEL_NAMES.WHISPER, device: 'webgpu' })
|
w.postMessage({ type: 'load-model', requestId, modelId: MODEL_NAMES.WHISPER, device: requestedDevice })
|
||||||
|
|
||||||
let readyResponse: any
|
let readyResponse: any
|
||||||
try {
|
try {
|
||||||
@@ -243,7 +278,7 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Capture actual device reported by the worker (may fall back to WASM)
|
// Capture actual device reported by the worker (may fall back to WASM)
|
||||||
const actualDevice = readyResponse?.device ?? 'webgpu'
|
const actualDevice = readyResponse?.device ?? requestedDevice
|
||||||
|
|
||||||
// Track GPU memory allocation
|
// Track GPU memory allocation
|
||||||
const coordinator = getGPUCoordinator()
|
const coordinator = getGPUCoordinator()
|
||||||
@@ -254,6 +289,7 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
|
|||||||
MODEL_VRAM_ESTIMATES[MODEL_NAMES.WHISPER] ?? 800 * 1024 * 1024,
|
MODEL_VRAM_ESTIMATES[MODEL_NAMES.WHISPER] ?? 800 * 1024 * 1024,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
lastManifest = { device: actualDevice }
|
||||||
state = 'ready'
|
state = 'ready'
|
||||||
updateInferenceStatus(MODEL_NAMES.WHISPER, { state: 'ready', device: actualDevice })
|
updateInferenceStatus(MODEL_NAMES.WHISPER, { state: 'ready', device: actualDevice })
|
||||||
onSuccess()
|
onSuccess()
|
||||||
@@ -317,5 +353,7 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
|
|||||||
terminate: terminateAdapter,
|
terminate: terminateAdapter,
|
||||||
onMessage,
|
onMessage,
|
||||||
get state() { return state },
|
get state() { return state },
|
||||||
|
get manifest() { return lastManifest },
|
||||||
|
get deviceLossCount() { return deviceLossCount },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,3 +53,15 @@ export const MAX_RESTARTS = 3
|
|||||||
|
|
||||||
/** Base delay in ms between restart attempts (multiplied by attempt number) */
|
/** Base delay in ms between restart attempts (multiplied by attempt number) */
|
||||||
export const RESTART_DELAY_MS = 1_000
|
export const RESTART_DELAY_MS = 1_000
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Device loss resilience
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Number of WebGPU device-loss events an adapter tolerates before proactively
|
||||||
|
* promoting subsequent loads to WASM. A single device loss may be transient
|
||||||
|
* (driver reset, GPU process crash), but repeated losses indicate the WebGPU
|
||||||
|
* path is unreliable on this device and WASM is safer.
|
||||||
|
*/
|
||||||
|
export const DEVICE_LOSS_WASM_THRESHOLD = 2
|
||||||
|
|||||||
@@ -91,4 +91,51 @@ describe('gpuResourceCoordinator', () => {
|
|||||||
coordinator.requestAllocation('model', 700 * 1024 * 1024)
|
coordinator.requestAllocation('model', 700 * 1024 * 1024)
|
||||||
expect(handler).not.toHaveBeenCalled()
|
expect(handler).not.toHaveBeenCalled()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('device loss telemetry', () => {
|
||||||
|
it('should start with zero device-loss metrics', () => {
|
||||||
|
const coordinator = createGPUResourceCoordinator(VRAM)
|
||||||
|
const metrics = coordinator.getDeviceLossMetrics()
|
||||||
|
|
||||||
|
expect(metrics.totalCount).toBe(0)
|
||||||
|
expect(metrics.byModel).toEqual({})
|
||||||
|
expect(metrics.lastEvent).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should aggregate device-loss events across models', () => {
|
||||||
|
const coordinator = createGPUResourceCoordinator(VRAM)
|
||||||
|
|
||||||
|
coordinator.recordDeviceLoss({ modelId: 'kokoro', reason: 'unknown', occurredAt: 100 })
|
||||||
|
coordinator.recordDeviceLoss({ modelId: 'kokoro', reason: 'unknown', occurredAt: 200 })
|
||||||
|
coordinator.recordDeviceLoss({ modelId: 'whisper', reason: 'destroyed', occurredAt: 300 })
|
||||||
|
|
||||||
|
const metrics = coordinator.getDeviceLossMetrics()
|
||||||
|
expect(metrics.totalCount).toBe(3)
|
||||||
|
expect(metrics.byModel).toEqual({ kokoro: 2, whisper: 1 })
|
||||||
|
expect(metrics.lastEvent).toEqual({ modelId: 'whisper', reason: 'destroyed', occurredAt: 300 })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should notify subscribers on device-loss events', () => {
|
||||||
|
const coordinator = createGPUResourceCoordinator(VRAM)
|
||||||
|
const handler = vi.fn()
|
||||||
|
coordinator.onDeviceLoss(handler)
|
||||||
|
|
||||||
|
const event = { modelId: 'kokoro', reason: 'unknown' as const, occurredAt: 100 }
|
||||||
|
coordinator.recordDeviceLoss(event)
|
||||||
|
|
||||||
|
expect(handler).toHaveBeenCalledTimes(1)
|
||||||
|
expect(handler).toHaveBeenCalledWith(event)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should allow unsubscribing from device-loss events', () => {
|
||||||
|
const coordinator = createGPUResourceCoordinator(VRAM)
|
||||||
|
const handler = vi.fn()
|
||||||
|
const unsub = coordinator.onDeviceLoss(handler)
|
||||||
|
|
||||||
|
unsub()
|
||||||
|
coordinator.recordDeviceLoss({ modelId: 'x', reason: 'unknown', occurredAt: 1 })
|
||||||
|
|
||||||
|
expect(handler).not.toHaveBeenCalled()
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -7,8 +7,13 @@
|
|||||||
*
|
*
|
||||||
* Emits memory pressure events when allocation nears the budget
|
* Emits memory pressure events when allocation nears the budget
|
||||||
* so consumers can decide to unload LRU models or fall back to WASM.
|
* so consumers can decide to unload LRU models or fall back to WASM.
|
||||||
|
*
|
||||||
|
* Also records device-loss telemetry so adapters can coordinate
|
||||||
|
* cross-model WASM fallback decisions.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import type { DeviceLossReason } from './protocol'
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Types
|
// Types
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -31,6 +36,21 @@ export interface GPUResourceUsage {
|
|||||||
models: string[]
|
models: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DeviceLossEvent {
|
||||||
|
modelId: string
|
||||||
|
reason: DeviceLossReason
|
||||||
|
occurredAt: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DeviceLossMetrics {
|
||||||
|
/** Total device-loss events recorded across all models */
|
||||||
|
totalCount: number
|
||||||
|
/** Per-model device-loss counts */
|
||||||
|
byModel: Record<string, number>
|
||||||
|
/** Most recent event, or null if none recorded */
|
||||||
|
lastEvent: DeviceLossEvent | null
|
||||||
|
}
|
||||||
|
|
||||||
export interface GPUResourceCoordinator {
|
export interface GPUResourceCoordinator {
|
||||||
/**
|
/**
|
||||||
* Request an allocation for a model.
|
* Request an allocation for a model.
|
||||||
@@ -58,6 +78,22 @@ export interface GPUResourceCoordinator {
|
|||||||
* Returns an unsubscribe function.
|
* Returns an unsubscribe function.
|
||||||
*/
|
*/
|
||||||
onMemoryPressure: (handler: (level: MemoryPressureLevel) => void) => () => void
|
onMemoryPressure: (handler: (level: MemoryPressureLevel) => void) => () => void
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a WebGPU device-loss event. Adapters call this from their error
|
||||||
|
* handlers when they detect a DEVICE_LOST error so the coordinator can
|
||||||
|
* maintain cross-model telemetry.
|
||||||
|
*/
|
||||||
|
recordDeviceLoss: (event: DeviceLossEvent) => void
|
||||||
|
|
||||||
|
/** Get current device-loss telemetry across all models */
|
||||||
|
getDeviceLossMetrics: () => DeviceLossMetrics
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Subscribe to device-loss events. Fired after `recordDeviceLoss()`.
|
||||||
|
* Returns an unsubscribe function.
|
||||||
|
*/
|
||||||
|
onDeviceLoss: (handler: (event: DeviceLossEvent) => void) => () => void
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -78,6 +114,10 @@ export function createGPUResourceCoordinator(
|
|||||||
const budget = estimatedVRAM > 0 ? estimatedVRAM * BUDGET_SAFETY_FACTOR : Number.POSITIVE_INFINITY
|
const budget = estimatedVRAM > 0 ? estimatedVRAM * BUDGET_SAFETY_FACTOR : Number.POSITIVE_INFINITY
|
||||||
const allocations = new Map<string, AllocationToken>()
|
const allocations = new Map<string, AllocationToken>()
|
||||||
const pressureHandlers = new Set<(level: MemoryPressureLevel) => void>()
|
const pressureHandlers = new Set<(level: MemoryPressureLevel) => void>()
|
||||||
|
const deviceLossHandlers = new Set<(event: DeviceLossEvent) => void>()
|
||||||
|
const deviceLossByModel = new Map<string, number>()
|
||||||
|
let deviceLossTotal = 0
|
||||||
|
let lastDeviceLossEvent: DeviceLossEvent | null = null
|
||||||
|
|
||||||
function getAllocated(): number {
|
function getAllocated(): number {
|
||||||
let total = 0
|
let total = 0
|
||||||
@@ -154,6 +194,27 @@ export function createGPUResourceCoordinator(
|
|||||||
return () => pressureHandlers.delete(handler)
|
return () => pressureHandlers.delete(handler)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordDeviceLoss(event: DeviceLossEvent): void {
|
||||||
|
deviceLossTotal++
|
||||||
|
deviceLossByModel.set(event.modelId, (deviceLossByModel.get(event.modelId) ?? 0) + 1)
|
||||||
|
lastDeviceLossEvent = event
|
||||||
|
for (const handler of deviceLossHandlers)
|
||||||
|
handler(event)
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDeviceLossMetrics(): DeviceLossMetrics {
|
||||||
|
return {
|
||||||
|
totalCount: deviceLossTotal,
|
||||||
|
byModel: Object.fromEntries(deviceLossByModel),
|
||||||
|
lastEvent: lastDeviceLossEvent,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onDeviceLoss(handler: (event: DeviceLossEvent) => void): () => void {
|
||||||
|
deviceLossHandlers.add(handler)
|
||||||
|
return () => deviceLossHandlers.delete(handler)
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
requestAllocation,
|
requestAllocation,
|
||||||
release,
|
release,
|
||||||
@@ -161,5 +222,8 @@ export function createGPUResourceCoordinator(
|
|||||||
getUsage,
|
getUsage,
|
||||||
getLRUModel,
|
getLRUModel,
|
||||||
onMemoryPressure,
|
onMemoryPressure,
|
||||||
|
recordDeviceLoss,
|
||||||
|
getDeviceLossMetrics,
|
||||||
|
onDeviceLoss,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { describe, expect, it } from 'vitest'
|
import { describe, expect, it } from 'vitest'
|
||||||
|
|
||||||
import { classifyError, isRecoverable } from './protocol'
|
import { classifyDeviceLossReason, classifyError, isRecoverable } from './protocol'
|
||||||
|
|
||||||
describe('classifyError', () => {
|
describe('classifyError', () => {
|
||||||
it('should classify OOM errors', () => {
|
it('should classify OOM errors', () => {
|
||||||
@@ -13,6 +13,16 @@ describe('classifyError', () => {
|
|||||||
expect(classifyError(new Error('WebGPU device lost unexpectedly'))).toBe('DEVICE_LOST')
|
expect(classifyError(new Error('WebGPU device lost unexpectedly'))).toBe('DEVICE_LOST')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should classify extended DEVICE_LOST patterns', () => {
|
||||||
|
expect(classifyError(new Error('GPU device lost'))).toBe('DEVICE_LOST')
|
||||||
|
expect(classifyError(new Error('GPUDevice was invalidated'))).toBe('DEVICE_LOST')
|
||||||
|
expect(classifyError(new Error('GPUDevice is invalid'))).toBe('DEVICE_LOST')
|
||||||
|
expect(classifyError(new Error('Device destroyed by user agent'))).toBe('DEVICE_LOST')
|
||||||
|
expect(classifyError(new Error('GPU process crashed'))).toBe('DEVICE_LOST')
|
||||||
|
expect(classifyError(new Error('GPU process lost'))).toBe('DEVICE_LOST')
|
||||||
|
expect(classifyError(new Error('WebGPU device is invalid'))).toBe('DEVICE_LOST')
|
||||||
|
})
|
||||||
|
|
||||||
it('should classify TIMEOUT errors', () => {
|
it('should classify TIMEOUT errors', () => {
|
||||||
expect(classifyError(new Error('operation timeout after 120s'))).toBe('TIMEOUT')
|
expect(classifyError(new Error('operation timeout after 120s'))).toBe('TIMEOUT')
|
||||||
})
|
})
|
||||||
@@ -70,3 +80,27 @@ describe('isRecoverable', () => {
|
|||||||
expect(isRecoverable('UNKNOWN')).toBe(false)
|
expect(isRecoverable('UNKNOWN')).toBe(false)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('classifyDeviceLossReason', () => {
|
||||||
|
it('should return destroyed for GPUDeviceLostInfo-shaped object', () => {
|
||||||
|
expect(classifyDeviceLossReason({ reason: 'destroyed', message: 'user requested' })).toBe('destroyed')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return unknown for non-destroyed structured reason', () => {
|
||||||
|
expect(classifyDeviceLossReason({ reason: 'unknown', message: 'driver reset' })).toBe('unknown')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return destroyed when error message contains "destroyed"', () => {
|
||||||
|
expect(classifyDeviceLossReason(new Error('Device destroyed by user agent'))).toBe('destroyed')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return unknown for generic device-loss messages', () => {
|
||||||
|
expect(classifyDeviceLossReason(new Error('GPU device lost'))).toBe('unknown')
|
||||||
|
expect(classifyDeviceLossReason(new Error('WebGPU device lost unexpectedly'))).toBe('unknown')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return unknown for non-device-loss inputs', () => {
|
||||||
|
expect(classifyDeviceLossReason(new Error('out of memory'))).toBe('unknown')
|
||||||
|
expect(classifyDeviceLossReason('random string')).toBe('unknown')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|||||||
@@ -148,6 +148,26 @@ export function createRequestId(): string {
|
|||||||
return `req_${Date.now().toString(36)}_${(counter++).toString(36)}`
|
return `req_${Date.now().toString(36)}_${(counter++).toString(36)}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NOTICE: Patterns observed in WebGPU device loss errors across Chromium,
|
||||||
|
// Firefox, Safari, and ONNX Runtime Web / transformers.js. Because we do not
|
||||||
|
// own the GPUDevice (it is created internally by transformers.js / ORT-web),
|
||||||
|
// we cannot attach a `device.lost` promise handler directly — string matching
|
||||||
|
// on thrown errors is the only available detection signal.
|
||||||
|
// References:
|
||||||
|
// - https://gpuweb.github.io/gpuweb/#gpudevicelostinfo
|
||||||
|
// - https://github.com/huggingface/transformers.js/issues/715
|
||||||
|
const DEVICE_LOSS_PATTERNS = [
|
||||||
|
'device was lost',
|
||||||
|
'device lost',
|
||||||
|
'gpu device lost',
|
||||||
|
'gpudevice was invalidated',
|
||||||
|
'gpudevice is invalid',
|
||||||
|
'device destroyed',
|
||||||
|
'gpu process crashed',
|
||||||
|
'gpu process lost',
|
||||||
|
'webgpu device is invalid',
|
||||||
|
] as const
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Classify an unknown error into an `InferenceErrorCode`.
|
* Classify an unknown error into an `InferenceErrorCode`.
|
||||||
* Used by worker adapters to normalise caught exceptions.
|
* Used by worker adapters to normalise caught exceptions.
|
||||||
@@ -162,7 +182,7 @@ export function classifyError(error: unknown, phase?: 'load' | 'inference'): Inf
|
|||||||
|
|
||||||
if (lower.includes('out of memory') || lower.includes('allocation failed'))
|
if (lower.includes('out of memory') || lower.includes('allocation failed'))
|
||||||
return 'OOM'
|
return 'OOM'
|
||||||
if (lower.includes('device was lost') || lower.includes('device lost'))
|
if (DEVICE_LOSS_PATTERNS.some(p => lower.includes(p)))
|
||||||
return 'DEVICE_LOST'
|
return 'DEVICE_LOST'
|
||||||
if (lower.includes('timeout'))
|
if (lower.includes('timeout'))
|
||||||
return 'TIMEOUT'
|
return 'TIMEOUT'
|
||||||
@@ -175,6 +195,31 @@ export function classifyError(error: unknown, phase?: 'load' | 'inference'): Inf
|
|||||||
return 'UNKNOWN'
|
return 'UNKNOWN'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Reason classification for a device-loss event, following `GPUDeviceLostInfo.reason`. */
|
||||||
|
export type DeviceLossReason = 'destroyed' | 'unknown'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Best-effort classification of a device-loss reason from an error message
|
||||||
|
* or a `GPUDeviceLostInfo`-shaped object. 'destroyed' implies intentional
|
||||||
|
* termination (no recovery); 'unknown' implies a transient event that may
|
||||||
|
* be recoverable via adapter restart or WASM fallback.
|
||||||
|
*/
|
||||||
|
export function classifyDeviceLossReason(error: unknown): DeviceLossReason {
|
||||||
|
// Prefer structured info when available (some browsers attach GPUDeviceLostInfo)
|
||||||
|
if (error && typeof error === 'object' && 'reason' in error) {
|
||||||
|
const reason = (error as { reason?: unknown }).reason
|
||||||
|
if (reason === 'destroyed')
|
||||||
|
return 'destroyed'
|
||||||
|
return 'unknown'
|
||||||
|
}
|
||||||
|
|
||||||
|
const msg = error instanceof Error ? error.message : String(error)
|
||||||
|
const lower = msg.toLowerCase()
|
||||||
|
if (lower.includes('destroyed'))
|
||||||
|
return 'destroyed'
|
||||||
|
return 'unknown'
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Determine whether an error code represents a potentially recoverable
|
* Determine whether an error code represents a potentially recoverable
|
||||||
* condition. TIMEOUT and DEVICE_LOST may succeed on retry (e.g. with
|
* condition. TIMEOUT and DEVICE_LOST may succeed on retry (e.g. with
|
||||||
|
|||||||
Generated
+2877
-3576
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -26,7 +26,7 @@ patchedDependencies:
|
|||||||
'@xsai/shared-chat@0.5.0-beta.2': patches/@xsai__shared-chat@0.5.0-beta.2.patch
|
'@xsai/shared-chat@0.5.0-beta.2': patches/@xsai__shared-chat@0.5.0-beta.2.patch
|
||||||
'@xsai/stream-text@0.5.0-beta.2': patches/@xsai__stream-text@0.5.0-beta.2.patch
|
'@xsai/stream-text@0.5.0-beta.2': patches/@xsai__stream-text@0.5.0-beta.2.patch
|
||||||
mineflayer-pathfinder: patches/mineflayer-pathfinder.patch
|
mineflayer-pathfinder: patches/mineflayer-pathfinder.patch
|
||||||
mineflayer@4.33.0: patches/mineflayer@4.33.0.patch
|
mineflayer@4.37.0: patches/mineflayer@4.37.0.patch
|
||||||
pixi-live2d-display: patches/pixi-live2d-display.patch
|
pixi-live2d-display: patches/pixi-live2d-display.patch
|
||||||
catalog:
|
catalog:
|
||||||
'@ax-llm/ax': ^19.0.43
|
'@ax-llm/ax': ^19.0.43
|
||||||
|
|||||||
Reference in New Issue
Block a user