feat(stage-ui): added AbortController for inference utils (#1682)

This commit is contained in:
NJX
2026-04-24 23:42:55 +08:00
committed by GitHub
parent d85709f900
commit 6a50da8e82
12 changed files with 733 additions and 89 deletions
@@ -37,7 +37,7 @@ export function useInferencePreload(options: UseInferencePreloadOptions = {}) {
await detectWebGPU()
const providersStore = useProvidersStore()
const tasks: { modelId: string, loader: () => Promise<void> }[] = []
const tasks: { modelId: string, loader: (signal: AbortSignal) => Promise<void> }[] = []
// Check if Kokoro TTS is configured
if (providersStore.configuredProviders['kokoro-local']) {
@@ -53,9 +53,9 @@ export function useInferencePreload(options: UseInferencePreloadOptions = {}) {
if (modelDef) {
tasks.push({
modelId: `kokoro-${modelDef.id}`,
loader: async () => {
loader: async (signal) => {
const adapter = await getKokoroAdapter()
await adapter.loadModel(modelDef.quantization, modelDef.platform)
await adapter.loadModel(modelDef.quantization, modelDef.platform, { signal })
},
})
}
@@ -5,6 +5,10 @@
* so they're ready before the user first needs them.
* Uses `setTimeout` to defer loading and avoid blocking the main
* thread during app startup.
*
* Cancellation uses an AbortController so the signal can be forwarded
* to adapter methods that accept `options.signal` — a cancelled preload
* aborts any in-flight model load instead of just ignoring the result.
*/
import { onUnmounted, ref } from 'vue'
@@ -12,8 +16,12 @@ import { onUnmounted, ref } from 'vue'
export interface PreloadTask {
/** Human-readable model name for logging */
modelId: string
/** The async function that loads the model */
loader: () => Promise<void>
/**
* The async function that loads the model. Receives an `AbortSignal`
* that will fire if the preload is cancelled; loaders should forward
* it to adapter methods (e.g. `adapter.loadModel(q, d, { signal })`).
*/
loader: (signal: AbortSignal) => Promise<void>
}
export interface UseModelPreloadOptions {
@@ -28,7 +36,7 @@ export function useModelPreload(options: UseModelPreloadOptions = {}) {
const preloadedModels = ref<string[]>([])
const failedModels = ref<string[]>([])
let cancelled = false
let abortController: AbortController | null = null
let timeoutId: ReturnType<typeof setTimeout> | undefined
/**
@@ -39,27 +47,38 @@ export function useModelPreload(options: UseModelPreloadOptions = {}) {
if (tasks.length === 0)
return
cancelled = false
// Fresh controller per scheduling — abort any prior in-flight preload first
if (abortController && !abortController.signal.aborted)
abortController.abort(new Error('Preload superseded by new schedule'))
abortController = new AbortController()
const signal = abortController.signal
timeoutId = setTimeout(async () => {
if (cancelled)
if (signal.aborted)
return
preloading.value = true
for (const task of tasks) {
if (cancelled)
if (signal.aborted)
break
try {
// eslint-disable-next-line no-console
console.debug(`[Preload] Loading ${task.modelId}...`)
await task.loader()
await task.loader(signal)
preloadedModels.value.push(task.modelId)
// eslint-disable-next-line no-console
console.debug(`[Preload] ${task.modelId} ready`)
}
catch (error) {
// AbortError is expected when the preload is cancelled — don't
// treat it as a failure.
if ((error as Error)?.name === 'AbortError') {
// eslint-disable-next-line no-console
console.debug(`[Preload] ${task.modelId} aborted`)
break
}
// Preload failures are non-fatal — model will load on first use
console.warn(`[Preload] ${task.modelId} failed:`, error)
failedModels.value.push(task.modelId)
@@ -71,7 +90,8 @@ export function useModelPreload(options: UseModelPreloadOptions = {}) {
}
function cancelPreload(): void {
cancelled = true
if (abortController && !abortController.signal.aborted)
abortController.abort(new Error('Preload cancelled'))
if (timeoutId !== undefined) {
clearTimeout(timeoutId)
timeoutId = undefined
@@ -16,7 +16,7 @@ import { removeInferenceStatus, updateInferenceStatus } from '../../../composabl
import { MODEL_IDS, MODEL_NAMES, TIMEOUTS } from '../constants'
import { getGPUCoordinator, getLoadQueue, MODEL_VRAM_ESTIMATES } from '../coordinator'
import { LOAD_PRIORITY } from '../load-queue'
import { createRequestId } from '../protocol'
import { createRequestId, InferenceAbortError, throwIfAborted } from '../protocol'
// ---------------------------------------------------------------------------
// Types
@@ -26,14 +26,22 @@ export interface BackgroundRemovalAdapter {
/**
* Load the background removal model in the worker.
* Must be called before `processImage()`.
* Pass `options.signal` to cancel; rejects with `InferenceAbortError`.
*/
load: (onProgress?: (p: ProgressPayload) => void) => Promise<void>
load: (
onProgress?: (p: ProgressPayload) => void,
options?: { signal?: AbortSignal },
) => Promise<void>
/**
* Remove the background from an image.
* Returns a new ImageData with the background alpha set to 0.
* Pass `options.signal` to cancel; rejects with `InferenceAbortError`.
*/
processImage: (imageData: ImageData) => Promise<ImageData>
processImage: (
imageData: ImageData,
options?: { signal?: AbortSignal },
) => Promise<ImageData>
/** Terminate the worker */
terminate: () => void
@@ -88,7 +96,8 @@ export function createBackgroundRemovalAdapter(): BackgroundRemovalAdapter {
/**
* Wait for a specific message type from the worker, filtered by requestId.
* Uses the unified protocol message types.
* Uses the unified protocol message types. Honors `signal` to cancel the
* wait (and notify the worker to discard the result).
*/
function waitForMessage<T = any>(
w: Worker,
@@ -96,25 +105,35 @@ export function createBackgroundRemovalAdapter(): BackgroundRemovalAdapter {
targetType: string,
timeout: number,
onOther?: (data: any) => void,
signal?: AbortSignal,
): Promise<T> {
return new Promise((resolve, reject) => {
let timeoutId: ReturnType<typeof setTimeout> | undefined
let abortListener: (() => void) | null = null
const handler = (event: MessageEvent) => {
const cleanup = (): void => {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
w.removeEventListener('message', handler)
if (abortListener && signal)
signal.removeEventListener('abort', abortListener)
}
const handler = (event: MessageEvent): void => {
if (event.data.requestId !== requestId)
return
if (event.data.type === targetType) {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
w.removeEventListener('message', handler)
cleanup()
resolve(event.data as T)
}
else if (event.data.type === 'error') {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
w.removeEventListener('message', handler)
reject(new Error(event.data.payload?.message ?? 'Worker error'))
cleanup()
const code = event.data.payload?.code
if (code === 'CANCELLED')
reject(new InferenceAbortError(event.data.payload?.message))
else
reject(new Error(event.data.payload?.message ?? 'Worker error'))
}
else {
onOther?.(event.data)
@@ -124,18 +143,40 @@ export function createBackgroundRemovalAdapter(): BackgroundRemovalAdapter {
w.addEventListener('message', handler)
timeoutId = setTimeout(() => {
w.removeEventListener('message', handler)
cleanup()
reject(new Error(`Background removal: timeout after ${timeout}ms`))
}, timeout)
if (signal) {
if (signal.aborted) {
cleanup()
w.postMessage({ type: 'cancel', requestId: createRequestId(), targetRequestId: requestId })
reject(new InferenceAbortError(typeof signal.reason === 'string' ? signal.reason : undefined))
return
}
abortListener = () => {
cleanup()
w.postMessage({ type: 'cancel', requestId: createRequestId(), targetRequestId: requestId })
const reason = signal.reason
reject(reason instanceof Error ? reason : new InferenceAbortError(typeof reason === 'string' ? reason : undefined))
}
signal.addEventListener('abort', abortListener)
}
})
}
async function load(onProgress?: (p: ProgressPayload) => void): Promise<void> {
async function load(
onProgress?: (p: ProgressPayload) => void,
options?: { signal?: AbortSignal },
): Promise<void> {
throwIfAborted(options?.signal)
return operationMutex.runExclusive(async () => {
throwIfAborted(options?.signal)
state = 'loading'
updateInferenceStatus(MODEL_NAMES.BG_REMOVAL, { state: 'downloading', device: 'webgpu' })
return getLoadQueue().enqueue(MODEL_NAMES.BG_REMOVAL, LOAD_PRIORITY.BACKGROUND_REMOVAL, async () => {
throwIfAborted(options?.signal)
const w = ensureWorker()
const requestId = createRequestId()
@@ -151,7 +192,7 @@ export function createBackgroundRemovalAdapter(): BackgroundRemovalAdapter {
total: payload.total,
})
}
})
}, options?.signal)
w.postMessage({ type: 'load-model', requestId, modelId: MODEL_IDS.BG_REMOVAL, device: 'webgpu' })
@@ -179,19 +220,31 @@ export function createBackgroundRemovalAdapter(): BackgroundRemovalAdapter {
state = 'ready'
updateInferenceStatus(MODEL_NAMES.BG_REMOVAL, { state: 'ready', device: actualDevice })
})
}, { signal: options?.signal })
})
}
async function processImage(imageData: ImageData): Promise<ImageData> {
async function processImage(
imageData: ImageData,
options?: { signal?: AbortSignal },
): Promise<ImageData> {
throwIfAborted(options?.signal)
return defaultPerfTracer.withMeasure('inference', 'bg-removal-process', () => operationMutex.runExclusive(async () => {
throwIfAborted(options?.signal)
if (!worker || (state !== 'ready' && state !== 'processing'))
throw new Error('Model not loaded. Call load() first.')
state = 'processing'
const requestId = createRequestId()
const resultPromise = waitForMessage<any>(worker, requestId, 'inference-result', PROCESS_TIMEOUT)
const resultPromise = waitForMessage<any>(
worker,
requestId,
'inference-result',
PROCESS_TIMEOUT,
undefined,
options?.signal,
)
// Send raw pixel data (transferable copy)
const pixelsCopy = new Uint8ClampedArray(imageData.data)
@@ -16,22 +16,36 @@ import { removeInferenceStatus, updateInferenceStatus } from '../../../composabl
import { DEVICE_LOSS_WASM_THRESHOLD, MAX_RESTARTS, MODEL_NAMES, RESTART_DELAY_MS, TIMEOUTS } from '../constants'
import { getGPUCoordinator, getLoadQueue, MODEL_VRAM_ESTIMATES } from '../coordinator'
import { LOAD_PRIORITY } from '../load-queue'
import { classifyDeviceLossReason, classifyError, createRequestId } from '../protocol'
import { classifyDeviceLossReason, classifyError, createRequestId, InferenceAbortError, throwIfAborted } from '../protocol'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface KokoroAdapter {
/** Load a TTS model with the given quantization and device */
/**
* Load a TTS model with the given quantization and device.
* Pass `options.signal` to cancel the load; the returned promise will
* reject with `InferenceAbortError` (name: `'AbortError'`).
*/
loadModel: (
quantization: string,
device: string,
options?: { onProgress?: (p: ProgressPayload) => void },
options?: {
onProgress?: (p: ProgressPayload) => void
signal?: AbortSignal
},
) => Promise<Voices>
/** Generate speech audio from text */
generate: (text: string, voice: VoiceKey) => Promise<ArrayBuffer>
/**
* Generate speech audio from text.
* Pass `options.signal` to cancel; rejects with `InferenceAbortError`.
*/
generate: (
text: string,
voice: VoiceKey,
options?: { signal?: AbortSignal },
) => Promise<ArrayBuffer>
/** Get the voices from the last loaded model */
getVoices: () => Voices
@@ -118,6 +132,10 @@ function writeString(view: DataView, offset: number, str: string): void {
/**
* Wait for a specific message type from the worker, filtered by requestId.
* Calls `callback` for interleaved messages (e.g. progress).
*
* If `signal` is provided and aborts, the returned Promise rejects with
* `InferenceAbortError` and a `cancel` message is sent to the worker so
* it can discard the result when it eventually arrives.
*/
function waitForWorkerMessage<T = any>(
worker: Worker,
@@ -125,25 +143,35 @@ function waitForWorkerMessage<T = any>(
targetType: string,
timeout: number,
callback?: (data: any) => void,
signal?: AbortSignal,
): Promise<T> {
return new Promise((resolve, reject) => {
let timeoutId: ReturnType<typeof setTimeout> | undefined
let abortListener: (() => void) | null = null
const handler = (event: MessageEvent) => {
const cleanup = (): void => {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
worker.removeEventListener('message', handler)
if (abortListener && signal)
signal.removeEventListener('abort', abortListener)
}
const handler = (event: MessageEvent): void => {
if (event.data.requestId !== requestId)
return
if (event.data.type === targetType) {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
worker.removeEventListener('message', handler)
cleanup()
resolve(event.data as T)
}
else if (event.data.type === 'error') {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
worker.removeEventListener('message', handler)
reject(new Error(event.data.payload?.message ?? 'Worker error'))
cleanup()
const code = event.data.payload?.code
if (code === 'CANCELLED')
reject(new InferenceAbortError(event.data.payload?.message))
else
reject(new Error(event.data.payload?.message ?? 'Worker error'))
}
else {
callback?.(event.data)
@@ -153,9 +181,26 @@ function waitForWorkerMessage<T = any>(
worker.addEventListener('message', handler)
timeoutId = setTimeout(() => {
worker.removeEventListener('message', handler)
cleanup()
reject(new Error(`Kokoro: timeout after ${timeout}ms waiting for '${targetType}'`))
}, timeout)
if (signal) {
if (signal.aborted) {
cleanup()
// Tell the worker to discard the result when it arrives
worker.postMessage({ type: 'cancel', requestId: createRequestId(), targetRequestId: requestId })
reject(new InferenceAbortError(typeof signal.reason === 'string' ? signal.reason : undefined))
return
}
abortListener = () => {
cleanup()
worker.postMessage({ type: 'cancel', requestId: createRequestId(), targetRequestId: requestId })
const reason = signal.reason
reject(reason instanceof Error ? reason : new InferenceAbortError(typeof reason === 'string' ? reason : undefined))
}
signal.addEventListener('abort', abortListener)
}
})
}
@@ -270,7 +315,10 @@ export function createKokoroAdapter(): KokoroAdapter {
async function loadModel(
quantization: string,
device: string,
options?: { onProgress?: (p: ProgressPayload) => void },
options?: {
onProgress?: (p: ProgressPayload) => void
signal?: AbortSignal
},
): Promise<Voices> {
// NOTICE: Proactive WASM promotion. If this adapter has suffered repeated
// WebGPU device-loss events, webgpu is unreliable on this device and we
@@ -287,10 +335,11 @@ export function createKokoroAdapter(): KokoroAdapter {
)
effectiveDevice = 'wasm'
}
throwIfAborted(options?.signal)
await ensureStarted()
return defaultPerfTracer.withMeasure('inference', 'kokoro-load-model', () => operationMutex.runExclusive(async () => {
throwIfAborted(options?.signal)
state = 'loading'
const modelStatusId = `kokoro-${quantization}`
@@ -303,7 +352,9 @@ export function createKokoroAdapter(): KokoroAdapter {
// Use the global load queue to serialize model loads across all adapters
return getLoadQueue().enqueue(modelStatusId, LOAD_PRIORITY.TTS, async () => {
throwIfAborted(options?.signal)
const requestId = createRequestId()
// Signal is also passed to the queue below for pending-entry removal
const readyPromise = waitForWorkerMessage<any>(worker!, requestId, 'model-ready', LOAD_MODEL_TIMEOUT, (data) => {
if (data.type === 'progress') {
@@ -320,7 +371,7 @@ export function createKokoroAdapter(): KokoroAdapter {
updateInferenceStatus(modelStatusId, { progress })
options?.onProgress?.(progress)
}
})
}, options?.signal)
worker!.postMessage({
type: 'load-model',
@@ -351,15 +402,25 @@ export function createKokoroAdapter(): KokoroAdapter {
if (!voices)
throw new Error('Kokoro worker did not return voice metadata')
return voices
})
}, { signal: options?.signal })
}), { quantization, device: effectiveDevice }).catch((error) => {
// Don't route AbortError through handleWorkerError — cancellation is
// not a worker failure and shouldn't trigger restart logic.
if ((error as Error)?.name === 'AbortError')
throw error
handleWorkerError(error instanceof Error ? error : new Error(String(error)))
throw error
})
}
async function generate(text: string, voice: VoiceKey): Promise<ArrayBuffer> {
async function generate(
text: string,
voice: VoiceKey,
options?: { signal?: AbortSignal },
): Promise<ArrayBuffer> {
throwIfAborted(options?.signal)
return defaultPerfTracer.withMeasure('inference', 'kokoro-generate', () => operationMutex.runExclusive(async () => {
throwIfAborted(options?.signal)
if (!worker)
throw new Error('Worker not initialized. Call loadModel() first.')
@@ -370,7 +431,14 @@ export function createKokoroAdapter(): KokoroAdapter {
state = 'running'
const requestId = createRequestId()
const resultPromise = waitForWorkerMessage<any>(worker, requestId, 'inference-result', GENERATE_TIMEOUT)
const resultPromise = waitForWorkerMessage<any>(
worker,
requestId,
'inference-result',
GENERATE_TIMEOUT,
undefined,
options?.signal,
)
worker.postMessage({
type: 'run-inference',
@@ -16,7 +16,7 @@ import { removeInferenceStatus, updateInferenceStatus } from '../../../composabl
import { DEVICE_LOSS_WASM_THRESHOLD, MAX_RESTARTS, MODEL_NAMES, RESTART_DELAY_MS, TIMEOUTS } from '../constants'
import { getGPUCoordinator, getLoadQueue, MODEL_VRAM_ESTIMATES } from '../coordinator'
import { LOAD_PRIORITY } from '../load-queue'
import { classifyDeviceLossReason, classifyError, createRequestId } from '../protocol'
import { classifyDeviceLossReason, classifyError, createRequestId, InferenceAbortError, throwIfAborted } from '../protocol'
// ---------------------------------------------------------------------------
// Types
@@ -47,11 +47,23 @@ export type WhisperEvent
| { type: 'error', payload: { code: string, message: string } }
export interface WhisperAdapter {
/** Load the Whisper model */
load: (onProgress?: (p: ProgressPayload) => void) => Promise<void>
/**
* Load the Whisper model.
* Pass `options.signal` to cancel the load; rejects with `InferenceAbortError`.
*/
load: (
onProgress?: (p: ProgressPayload) => void,
options?: { signal?: AbortSignal },
) => Promise<void>
/** Transcribe audio, returning the text result */
transcribe: (input: WhisperTranscribeInput) => Promise<string>
/**
* Transcribe audio, returning the text result.
* Pass `options.signal` to cancel; rejects with `InferenceAbortError`.
*/
transcribe: (
input: WhisperTranscribeInput,
options?: { signal?: AbortSignal },
) => Promise<string>
/** Terminate the worker */
terminate: () => void
@@ -188,6 +200,8 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
/**
* Wait for a specific unified protocol message type, filtered by requestId.
* If `signal` is provided and aborts, sends a `cancel` message to the
* worker and rejects with `InferenceAbortError`.
*/
function waitForMessage<T = any>(
w: Worker,
@@ -195,25 +209,35 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
targetType: string,
timeout: number,
onOther?: (data: any) => void,
signal?: AbortSignal,
): Promise<T> {
return new Promise((resolve, reject) => {
let timeoutId: ReturnType<typeof setTimeout> | undefined
let abortListener: (() => void) | null = null
const handler = (event: MessageEvent) => {
const cleanup = (): void => {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
w.removeEventListener('message', handler)
if (abortListener && signal)
signal.removeEventListener('abort', abortListener)
}
const handler = (event: MessageEvent): void => {
if (event.data.requestId !== requestId)
return
if (event.data.type === targetType) {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
w.removeEventListener('message', handler)
cleanup()
resolve(event.data as T)
}
else if (event.data.type === 'error') {
if (timeoutId !== undefined)
clearTimeout(timeoutId)
w.removeEventListener('message', handler)
reject(new Error(event.data.payload?.message ?? 'Worker error'))
cleanup()
const code = event.data.payload?.code
if (code === 'CANCELLED')
reject(new InferenceAbortError(event.data.payload?.message))
else
reject(new Error(event.data.payload?.message ?? 'Worker error'))
}
else {
onOther?.(event.data)
@@ -223,14 +247,31 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
w.addEventListener('message', handler)
timeoutId = setTimeout(() => {
w.removeEventListener('message', handler)
cleanup()
reject(new Error(`Whisper: timeout after ${timeout}ms waiting for '${targetType}'`))
}, timeout)
if (signal) {
if (signal.aborted) {
cleanup()
w.postMessage({ type: 'cancel', requestId: createRequestId(), targetRequestId: requestId })
reject(new InferenceAbortError(typeof signal.reason === 'string' ? signal.reason : undefined))
return
}
abortListener = () => {
cleanup()
w.postMessage({ type: 'cancel', requestId: createRequestId(), targetRequestId: requestId })
const reason = signal.reason
reject(reason instanceof Error ? reason : new InferenceAbortError(typeof reason === 'string' ? reason : undefined))
}
signal.addEventListener('abort', abortListener)
}
})
}
async function load(
onProgress?: (p: ProgressPayload) => void,
options?: { signal?: AbortSignal },
): Promise<void> {
// NOTICE: Proactive WASM promotion after repeated device-loss events.
// See kokoro.ts for rationale. Whisper always requests 'webgpu' from the
@@ -242,12 +283,14 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
+ `promoting load from webgpu to wasm.`,
)
}
throwIfAborted(options?.signal)
return operationMutex.runExclusive(async () => {
throwIfAborted(options?.signal)
state = 'loading'
updateInferenceStatus(MODEL_NAMES.WHISPER, { state: 'downloading', device: requestedDevice as any })
return getLoadQueue().enqueue(MODEL_NAMES.WHISPER, LOAD_PRIORITY.ASR, async () => {
throwIfAborted(options?.signal)
const w = ensureWorker()
const requestId = createRequestId()
@@ -263,7 +306,7 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
total: payload.total,
})
}
})
}, options?.signal)
w.postMessage({ type: 'load-model', requestId, modelId: MODEL_NAMES.WHISPER, device: requestedDevice })
@@ -293,19 +336,31 @@ export function createWhisperAdapter(workerUrl: string | URL): WhisperAdapter {
state = 'ready'
updateInferenceStatus(MODEL_NAMES.WHISPER, { state: 'ready', device: actualDevice })
onSuccess()
})
}, { signal: options?.signal })
})
}
async function transcribe(input: WhisperTranscribeInput): Promise<string> {
async function transcribe(
input: WhisperTranscribeInput,
options?: { signal?: AbortSignal },
): Promise<string> {
throwIfAborted(options?.signal)
return defaultPerfTracer.withMeasure('inference', 'whisper-transcribe', () => operationMutex.runExclusive(async () => {
throwIfAborted(options?.signal)
if (!worker || state !== 'ready')
throw new Error('Model not loaded. Call load() first.')
state = 'transcribing'
const requestId = createRequestId()
const resultPromise = waitForMessage<any>(worker, requestId, 'inference-result', TRANSCRIBE_TIMEOUT)
const resultPromise = waitForMessage<any>(
worker,
requestId,
'inference-result',
TRANSCRIBE_TIMEOUT,
undefined,
options?.signal,
)
worker.postMessage({
type: 'run-inference',
@@ -91,4 +91,107 @@ describe('loadQueue', () => {
await p
expect(queue.active).toBeNull()
})
describe('cancellation', () => {
it('should reject immediately if signal is already aborted', async () => {
const queue = createLoadQueue()
const controller = new AbortController()
controller.abort()
let loaderCalled = false
const promise = queue.enqueue(
'already-aborted',
1,
async () => {
loaderCalled = true
return 'x'
},
{ signal: controller.signal },
)
await expect(promise).rejects.toMatchObject({ name: 'AbortError' })
expect(loaderCalled).toBe(false)
})
it('should remove a pending entry from the queue when its signal aborts', async () => {
const queue = createLoadQueue()
// Hold the queue with a slow loader
let releaseHold!: () => void
const hold = queue.enqueue('hold', 10, () => new Promise<void>(r => releaseHold = r))
const controller = new AbortController()
let loaderCalled = false
const pending = queue.enqueue(
'pending',
1,
async () => {
loaderCalled = true
},
{ signal: controller.signal },
)
expect(queue.pending).toContain('pending')
controller.abort(new Error('cancelled by test'))
await expect(pending).rejects.toThrow('cancelled by test')
expect(queue.pending).not.toContain('pending')
expect(loaderCalled).toBe(false)
releaseHold()
await hold
})
it('should not interrupt an active loader — that is the loader\'s responsibility', async () => {
const queue = createLoadQueue()
const controller = new AbortController()
// Loader that honors its own signal
const activePromise = queue.enqueue(
'active',
1,
async () => {
await new Promise((resolve, reject) => {
controller.signal.addEventListener('abort', () => {
reject(new Error('loader aborted'))
})
})
},
{ signal: controller.signal },
)
// Give the loader a tick to start
await new Promise(r => setTimeout(r, 5))
expect(queue.active).toBe('active')
controller.abort()
await expect(activePromise).rejects.toThrow('loader aborted')
})
it('should recover after a cancelled entry and continue processing subsequent items', async () => {
const queue = createLoadQueue()
// Hold the queue
let releaseHold!: () => void
const hold = queue.enqueue('hold', 10, () => new Promise<void>(r => releaseHold = r))
const controller = new AbortController()
const cancelled = queue.enqueue(
'cancelled',
5,
async () => 'should-not-run',
{ signal: controller.signal },
)
const later = queue.enqueue('later', 5, async () => 'later-result')
controller.abort()
await expect(cancelled).rejects.toMatchObject({ name: 'AbortError' })
releaseHold()
await hold
expect(await later).toBe('later-result')
})
})
})
@@ -6,8 +6,17 @@
* are dequeued first.
*
* Default priorities: TTS = 10, ASR = 5, BackgroundRemoval = 1.
*
* Cancellation: pass an `AbortSignal` in `enqueueOptions` to `enqueue()`.
* When aborted, the entry is removed from the pending queue (if not yet
* active) and its promise is rejected with `InferenceAbortError`. If the
* entry is already running, the loader itself is responsible for honoring
* the same signal and rejecting accordingly — the queue cannot interrupt
* an in-flight async loader.
*/
import { InferenceAbortError } from './protocol'
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
@@ -18,6 +27,13 @@ interface QueueEntry<T> {
loader: () => Promise<T>
resolve: (value: T) => void
reject: (error: unknown) => void
signal?: AbortSignal
abortHandler?: () => void
}
export interface EnqueueOptions {
/** Abort the enqueued load. Rejects the returned promise with `InferenceAbortError`. */
signal?: AbortSignal
}
export interface LoadQueue {
@@ -26,7 +42,12 @@ export interface LoadQueue {
* the loader completes. If another load is in progress, this
* one waits in a priority queue.
*/
enqueue: <T>(modelId: string, priority: number, loader: () => Promise<T>) => Promise<T>
enqueue: <T>(
modelId: string,
priority: number,
loader: () => Promise<T>,
options?: EnqueueOptions,
) => Promise<T>
/** Model IDs waiting in the queue */
readonly pending: string[]
@@ -44,6 +65,13 @@ export function createLoadQueue(): LoadQueue {
let active: string | null = null
let running = false
function detachAbortHandler(entry: QueueEntry<any>): void {
if (entry.signal && entry.abortHandler) {
entry.signal.removeEventListener('abort', entry.abortHandler)
entry.abortHandler = undefined
}
}
async function processQueue(): Promise<void> {
if (running)
return
@@ -54,12 +82,24 @@ export function createLoadQueue(): LoadQueue {
queue.sort((a, b) => b.priority - a.priority)
const entry = queue.shift()!
// Skip already-aborted entries (the abort handler may have fired
// before this dequeue; it removes the entry from the array but we
// also guard here in case of races)
if (entry.signal?.aborted) {
detachAbortHandler(entry)
const reason = entry.signal.reason
entry.reject(reason instanceof Error ? reason : new InferenceAbortError())
continue
}
active = entry.modelId
try {
const result = await entry.loader()
detachAbortHandler(entry)
entry.resolve(result)
}
catch (error) {
detachAbortHandler(entry)
entry.reject(error)
}
}
@@ -72,9 +112,39 @@ export function createLoadQueue(): LoadQueue {
modelId: string,
priority: number,
loader: () => Promise<T>,
options?: EnqueueOptions,
): Promise<T> {
return new Promise<T>((resolve, reject) => {
queue.push({ modelId, priority, loader, resolve, reject })
const entry: QueueEntry<T> = {
modelId,
priority,
loader,
resolve,
reject,
signal: options?.signal,
}
if (options?.signal) {
if (options.signal.aborted) {
const reason = options.signal.reason
reject(reason instanceof Error ? reason : new InferenceAbortError())
return
}
entry.abortHandler = () => {
// Remove from pending queue if still there. If the entry has
// already been dequeued (active load), the loader's own abort
// propagation will handle rejection.
const idx = queue.indexOf(entry)
if (idx >= 0) {
queue.splice(idx, 1)
const reason = options.signal!.reason
reject(reason instanceof Error ? reason : new InferenceAbortError())
}
}
options.signal.addEventListener('abort', entry.abortHandler)
}
queue.push(entry)
processQueue()
})
}
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest'
import { classifyDeviceLossReason, classifyError, isRecoverable } from './protocol'
import { classifyDeviceLossReason, classifyError, InferenceAbortError, isRecoverable, throwIfAborted } from './protocol'
describe('classifyError', () => {
it('should classify OOM errors', () => {
@@ -79,6 +79,74 @@ describe('isRecoverable', () => {
it('should mark UNKNOWN as not recoverable', () => {
expect(isRecoverable('UNKNOWN')).toBe(false)
})
it('should mark CANCELLED as not recoverable', () => {
expect(isRecoverable('CANCELLED')).toBe(false)
})
})
describe('inferenceAbortError', () => {
it('should have name "AbortError" for DOM compatibility', () => {
const err = new InferenceAbortError()
expect(err.name).toBe('AbortError')
})
it('should carry code "CANCELLED"', () => {
const err = new InferenceAbortError()
expect(err.code).toBe('CANCELLED')
})
it('should accept a custom message', () => {
const err = new InferenceAbortError('user cancelled')
expect(err.message).toBe('user cancelled')
})
it('should use a default message when none provided', () => {
const err = new InferenceAbortError()
expect(err.message).toBe('The operation was aborted')
})
it('should be instanceof Error', () => {
expect(new InferenceAbortError()).toBeInstanceOf(Error)
})
})
describe('throwIfAborted', () => {
it('should be a no-op when signal is undefined', () => {
expect(() => throwIfAborted(undefined)).not.toThrow()
})
it('should be a no-op when signal is not aborted', () => {
const controller = new AbortController()
expect(() => throwIfAborted(controller.signal)).not.toThrow()
})
it('should throw when signal is already aborted', () => {
const controller = new AbortController()
controller.abort()
expect(() => throwIfAborted(controller.signal)).toThrow()
})
it('should throw the signal\'s reason if it is an Error', () => {
const controller = new AbortController()
const reason = new Error('custom reason')
controller.abort(reason)
expect(() => throwIfAborted(controller.signal)).toThrow(reason)
})
it('should throw InferenceAbortError when reason is a string', () => {
const controller = new AbortController()
controller.abort('cancelled by string')
try {
throwIfAborted(controller.signal)
}
catch (err) {
expect((err as Error).name).toBe('AbortError')
expect((err as Error).message).toBe('cancelled by string')
return
}
throw new Error('should have thrown')
})
})
describe('classifyDeviceLossReason', () => {
@@ -53,6 +53,7 @@ export type InferenceErrorCode
| 'DEVICE_LOST'
| 'LOAD_FAILED'
| 'INFERENCE_FAILED'
| 'CANCELLED'
| 'UNKNOWN'
export interface ErrorPayload {
@@ -87,10 +88,31 @@ export interface UnloadModelRequest {
requestId: string
}
/**
* Cancel an in-flight or queued request. The worker should stop any
* ongoing work tied to `targetRequestId` and must NOT send a normal
* `model-ready` / `inference-result` response for that request; instead
* it should send an `ErrorResponse` with code `'CANCELLED'` so the
* adapter can reject the caller's promise deterministically.
*
* NOTE: Cancellation is best-effort. We cannot interrupt a synchronous
* transformers.js / ONNX Runtime call that is already executing on the
* worker thread. What the cancel signal does guarantee is that the
* adapter stops waiting and the worker discards the result when it
* eventually arrives.
*/
export interface CancelRequest {
type: 'cancel'
requestId: string
/** The requestId of the operation to cancel */
targetRequestId: string
}
export type WorkerInboundMessage<TInput = unknown>
= | LoadModelRequest
| RunInferenceRequest<TInput>
| UnloadModelRequest
| CancelRequest
// ---------------------------------------------------------------------------
// Worker → Main responses
@@ -228,3 +250,27 @@ export function classifyDeviceLossReason(error: unknown): DeviceLossReason {
export function isRecoverable(code: InferenceErrorCode): boolean {
return code === 'TIMEOUT' || code === 'DEVICE_LOST'
}
/**
* Canonical error thrown by inference adapters when an operation is
* cancelled via AbortSignal. Matches the DOM convention of `name === 'AbortError'`
* so existing `if (err.name === 'AbortError')` checks work unchanged.
*/
export class InferenceAbortError extends Error {
override readonly name = 'AbortError'
readonly code = 'CANCELLED' as const
constructor(message = 'The operation was aborted') {
super(message)
}
}
/** Throw `InferenceAbortError` if the signal is already aborted. */
export function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) {
const reason = signal.reason
if (reason instanceof Error)
throw reason
throw new InferenceAbortError(typeof reason === 'string' ? reason : undefined)
}
}
+69 -13
View File
@@ -174,6 +174,42 @@ async function base64ToFeatures(base64Audio: string): Promise<Float32Array> {
// Helpers
// ---------------------------------------------------------------------------
/**
* RequestIds the main thread has asked us to cancel. When an in-flight
* operation resolves, we check this set before posting the result; if
* the id is present, we send a `CANCELLED` error instead so the adapter
* rejects the caller's promise deterministically.
*
* We cannot synchronously interrupt a transformers.js call already running
* on this thread (no abort primitive is exposed) — cancellation here is
* about not leaking the stale result, not about stopping GPU work.
*/
const cancelledRequestIds = new Set<string>()
function markCancelled(targetRequestId: string): void {
cancelledRequestIds.add(targetRequestId)
// Emit the error now so the adapter can resolve immediately even if
// the inference keeps running in the background.
const msg: ErrorResponse = {
type: 'error',
requestId: targetRequestId,
payload: {
code: 'CANCELLED',
message: 'Operation cancelled by caller',
recoverable: false,
},
}
globalThis.postMessage(msg)
}
function isCancelled(requestId: string): boolean {
return cancelledRequestIds.has(requestId)
}
function clearCancelled(requestId: string): void {
cancelledRequestIds.delete(requestId)
}
function sendProgress(requestId: string, phase: 'download' | 'compile' | 'warmup' | 'inference', percent: number, message?: string, extra?: Record<string, unknown>): void {
const msg: ProgressResponse = {
type: 'progress',
@@ -243,16 +279,25 @@ async function loadModel(request: LoadModelRequest): Promise<void> {
max_new_tokens: 1,
} as Record<string, unknown>)
const ready: ModelReadyResponse = {
type: 'model-ready',
requestId,
modelId: MODEL_NAMES.WHISPER,
device: resolvedDevice,
if (isCancelled(requestId)) {
// Adapter already received a CANCELLED error; drop the stale result.
clearCancelled(requestId)
}
else {
const ready: ModelReadyResponse = {
type: 'model-ready',
requestId,
modelId: MODEL_NAMES.WHISPER,
device: resolvedDevice,
}
globalThis.postMessage(ready)
}
globalThis.postMessage(ready)
}
catch (error) {
sendError(requestId, error, 'load')
if (isCancelled(requestId))
clearCancelled(requestId)
else
sendError(requestId, error, 'load')
}
finally {
currentLoadRequestId = null
@@ -311,15 +356,23 @@ async function runInference(request: RunInferenceRequest<WhisperInput>): Promise
const outputText = tokenizer.batch_decode(outputs as Tensor, { skip_special_tokens: true })
const result: InferenceResultResponse<WhisperOutput> = {
type: 'inference-result',
requestId,
output: { text: outputText },
if (isCancelled(requestId)) {
clearCancelled(requestId)
}
else {
const result: InferenceResultResponse<WhisperOutput> = {
type: 'inference-result',
requestId,
output: { text: outputText },
}
globalThis.postMessage(result)
}
globalThis.postMessage(result)
}
catch (error) {
sendError(requestId, error, 'inference')
if (isCancelled(requestId))
clearCancelled(requestId)
else
sendError(requestId, error, 'inference')
}
finally {
processing = false
@@ -344,5 +397,8 @@ globalThis.addEventListener('message', async (event: MessageEvent<WorkerInboundM
// Whisper uses singleton pattern — can't fully unload, but acknowledge
globalThis.postMessage({ type: 'model-unloaded', requestId: message.requestId })
break
case 'cancel':
markCancelled(message.targetRequestId)
break
}
})
@@ -75,6 +75,31 @@ function sendError(requestId: string, error: unknown, phase?: 'load' | 'inferenc
globalThis.postMessage(msg)
}
// NOTICE: Cancellation tracking — see Whisper worker for rationale.
const cancelledRequestIds = new Set<string>()
function markCancelled(targetRequestId: string): void {
cancelledRequestIds.add(targetRequestId)
const msg: ErrorResponse = {
type: 'error',
requestId: targetRequestId,
payload: {
code: 'CANCELLED',
message: 'Operation cancelled by caller',
recoverable: false,
},
}
globalThis.postMessage(msg)
}
function isCancelled(requestId: string): boolean {
return cancelledRequestIds.has(requestId)
}
function clearCancelled(requestId: string): void {
cancelledRequestIds.delete(requestId)
}
/**
* Detect whether WebGPU is available inside the worker.
*/
@@ -97,6 +122,10 @@ async function loadModel(request: LoadModelRequest): Promise<void> {
try {
if (model && processor) {
if (isCancelled(requestId)) {
clearCancelled(requestId)
return
}
const ready: ModelReadyResponse = {
type: 'model-ready',
requestId,
@@ -129,6 +158,11 @@ async function loadModel(request: LoadModelRequest): Promise<void> {
processor = await AutoProcessor.from_pretrained(MODEL_ID, {})
if (isCancelled(requestId)) {
clearCancelled(requestId)
return
}
const ready: ModelReadyResponse = {
type: 'model-ready',
requestId,
@@ -138,7 +172,10 @@ async function loadModel(request: LoadModelRequest): Promise<void> {
globalThis.postMessage(ready)
}
catch (error) {
sendError(requestId, error, 'load')
if (isCancelled(requestId))
clearCancelled(requestId)
else
sendError(requestId, error, 'load')
}
}
@@ -169,6 +206,11 @@ async function runInference(request: RunInferenceRequest<BackgroundRemovalInput>
output[0].mul(255).to('uint8'),
).resize(width, height)
if (isCancelled(requestId)) {
clearCancelled(requestId)
return
}
const maskData = new Uint8Array(mask.data.buffer)
const result: InferenceResultResponse<BackgroundRemovalOutput> = {
@@ -180,7 +222,10 @@ async function runInference(request: RunInferenceRequest<BackgroundRemovalInput>
;(globalThis as any).postMessage(result, [maskData.buffer])
}
catch (error) {
sendError(requestId, error, 'inference')
if (isCancelled(requestId))
clearCancelled(requestId)
else
sendError(requestId, error, 'inference')
}
}
@@ -203,5 +248,8 @@ globalThis.addEventListener('message', async (event: MessageEvent<WorkerInboundM
processor = null
globalThis.postMessage({ type: 'model-unloaded', requestId: message.requestId })
break
case 'cancel':
markCancelled(message.targetRequestId)
break
}
})
+60 -3
View File
@@ -74,6 +74,33 @@ const DEVICE_FALLBACK: Record<string, string[]> = {
cpu: [],
}
// NOTICE: Cancellation tracking — see Whisper worker for the full rationale.
// We cannot interrupt a transformers.js call synchronously; this set lets us
// drop stale results when they arrive.
const cancelledRequestIds = new Set<string>()
function markCancelled(targetRequestId: string): void {
cancelledRequestIds.add(targetRequestId)
const msg: ErrorResponse = {
type: 'error',
requestId: targetRequestId,
payload: {
code: 'CANCELLED',
message: 'Operation cancelled by caller',
recoverable: false,
},
}
globalThis.postMessage(msg)
}
function isCancelled(requestId: string): boolean {
return cancelledRequestIds.has(requestId)
}
function clearCancelled(requestId: string): void {
cancelledRequestIds.delete(requestId)
}
function sendError(requestId: string, error: unknown, phase?: 'load' | 'inference'): void {
const message = error instanceof Error ? error.message : String(error)
const code = classifyError(error, phase)
@@ -96,6 +123,10 @@ async function loadModel(request: LoadModelRequest): Promise<void> {
try {
// Check if we already have the correct model loaded
if (ttsModel && currentQuantization === quantization && currentDevice === device) {
if (isCancelled(requestId)) {
clearCancelled(requestId)
return
}
const ready: ModelReadyResponse = {
type: 'model-ready',
requestId,
@@ -151,6 +182,10 @@ async function loadModel(request: LoadModelRequest): Promise<void> {
currentQuantization = quantization
currentDevice = attempt.device
if (isCancelled(requestId)) {
clearCancelled(requestId)
return
}
const ready: ModelReadyResponse = {
type: 'model-ready',
requestId,
@@ -175,10 +210,16 @@ async function loadModel(request: LoadModelRequest): Promise<void> {
}
// All attempts exhausted
sendError(requestId, lastError ?? new Error('All dtype/device combinations failed'), 'load')
if (isCancelled(requestId))
clearCancelled(requestId)
else
sendError(requestId, lastError ?? new Error('All dtype/device combinations failed'), 'load')
}
catch (error) {
sendError(requestId, error, 'load')
if (isCancelled(requestId))
clearCancelled(requestId)
else
sendError(requestId, error, 'load')
}
}
@@ -190,6 +231,11 @@ async function runInference(request: RunInferenceRequest<KokoroInferenceInput>):
if (!ttsModel)
throw new Error('Model not loaded. Send load-model first.')
if (isCancelled(requestId)) {
clearCancelled(requestId)
return
}
const result: InferenceResultResponse<KokoroVoicesOutput> = {
type: 'inference-result',
requestId,
@@ -206,6 +252,11 @@ async function runInference(request: RunInferenceRequest<KokoroInferenceInput>):
const { text, voice } = input
const audioResult = await ttsModel.generate(text, { voice })
if (isCancelled(requestId)) {
clearCancelled(requestId)
return
}
// Transfer raw PCM Float32Array directly — avoids WAV blob encode/decode overhead.
const samples = audioResult.audio
const result: InferenceResultResponse<KokoroGenerateOutput> = {
@@ -216,7 +267,10 @@ async function runInference(request: RunInferenceRequest<KokoroInferenceInput>):
;(globalThis as any).postMessage(result, [samples.buffer])
}
catch (error) {
sendError(requestId, error, 'inference')
if (isCancelled(requestId))
clearCancelled(requestId)
else
sendError(requestId, error, 'inference')
}
}
@@ -240,6 +294,9 @@ globalThis.addEventListener('message', async (event: MessageEvent<WorkerInboundM
currentDevice = null
globalThis.postMessage({ type: 'model-unloaded', requestId: message.requestId })
break
case 'cancel':
markCancelled(message.targetRequestId)
break
default:
console.warn('[Kokoro Worker] Unknown message type:', (message as any).type)
}