From 4b0577ddcf4472021d21b4ef84146e1591f98c0b Mon Sep 17 00:00:00 2001 From: Neko Date: Fri, 25 Jul 2025 15:12:50 +0800 Subject: [PATCH] refactor(stage-tamagotchi,crates): no more cuda required and candle removed (#323) --- .github/workflows/release-tamagotchi.yml | 44 +- Cargo.lock | 830 +----------------- Cargo.toml | 4 +- apps/stage-tamagotchi/src-tauri/Cargo.toml | 5 +- .../src-tauri/capabilities/default.json | 2 +- apps/stage-tamagotchi/src-tauri/src/lib.rs | 2 +- .../src/composables/audio/extract-whisper.ts | 7 +- .../stage-tamagotchi/src/composables/tauri.ts | 2 +- apps/stage-tamagotchi/src/pages/index.vue | 4 +- apps/stage-tamagotchi/src/tauri/invoke.ts | 4 +- .../Cargo.toml | 62 -- .../build.rs | 5 - .../src/models/mod.rs | 24 - .../src/models/whisper/languages.rs | 102 --- .../src/models/whisper/mod.rs | 478 ---------- .../Cargo.lock | 0 .../Cargo.toml | 49 ++ .../build.rs | 5 + .../commands/ipc_audio_transcription.toml | 0 .../commands/load_candle_model_whisper.toml | 0 .../commands/load_ort_model_whisper.toml | 13 + .../permissions/autogenerated/reference.md | 36 +- .../permissions/default.toml | 2 +- .../permissions/schemas/schema.json | 16 +- .../src/helpers/huggingface.rs | 17 +- .../src/helpers/mod.rs | 0 .../src/lib.rs | 26 +- .../src/models/mod.rs | 27 + .../src/models/whisper/melfilters.bytes | Bin .../src/models/whisper/melfilters128.bytes | Bin .../src/models/whisper/mod.rs | 2 + .../src/models/whisper/whisper.rs | 507 +++++++++++ .../src/models/whisper/whisper_processor.rs | 134 +++ .../tauri-plugin-ipc-audio-vad-ort/Cargo.toml | 4 +- 34 files changed, 827 insertions(+), 1586 deletions(-) delete mode 100644 crates/tauri-plugin-ipc-audio-transcription-candle/Cargo.toml delete mode 100644 crates/tauri-plugin-ipc-audio-transcription-candle/build.rs delete mode 100644 crates/tauri-plugin-ipc-audio-transcription-candle/src/models/mod.rs delete mode 100644 crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/languages.rs delete mode 100644 crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/mod.rs rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/Cargo.lock (100%) create mode 100644 crates/tauri-plugin-ipc-audio-transcription-ort/Cargo.toml create mode 100644 crates/tauri-plugin-ipc-audio-transcription-ort/build.rs rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/permissions/autogenerated/commands/ipc_audio_transcription.toml (100%) rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/permissions/autogenerated/commands/load_candle_model_whisper.toml (100%) create mode 100644 crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/load_ort_model_whisper.toml rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/permissions/autogenerated/reference.md (57%) rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/permissions/default.toml (88%) rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/permissions/schemas/schema.json (92%) rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/src/helpers/huggingface.rs (86%) rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/src/helpers/mod.rs (100%) rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/src/lib.rs (80%) create mode 100644 crates/tauri-plugin-ipc-audio-transcription-ort/src/models/mod.rs rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/src/models/whisper/melfilters.bytes (100%) rename crates/{tauri-plugin-ipc-audio-transcription-candle => tauri-plugin-ipc-audio-transcription-ort}/src/models/whisper/melfilters128.bytes (100%) create mode 100644 crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/mod.rs create mode 100644 crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper.rs create mode 100644 crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper_processor.rs diff --git a/.github/workflows/release-tamagotchi.yml b/.github/workflows/release-tamagotchi.yml index 3f36658c1..b4e949ce0 100644 --- a/.github/workflows/release-tamagotchi.yml +++ b/.github/workflows/release-tamagotchi.yml @@ -112,13 +112,6 @@ jobs: with: arch: x64 - - name: Install CUDA Toolkit (Linux and Windows Only) - uses: Jimver/cuda-toolkit@v0.2.24 - if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' || matrix.os == 'windows-latest' - with: - cuda: '12.5.0' - log-file-suffix: ${{matrix.os}}.txt - - name: Install dependencies run: pnpm install --frozen-lockfile @@ -131,44 +124,9 @@ jobs: - name: Build Application (Linux and Windows Only) if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' || matrix.os == 'windows-latest' - run: pnpm run build:tamagotchi --target ${{ matrix.target }} --features cuda --verbose + run: pnpm run build:tamagotchi --target ${{ matrix.target }} --verbose env: RUST_BACKTRACE: '1' - # Call to `nvcc` (which part of the CUDA Toolkit) doesn't require a physical GPU, yet bindgen_cuda requires `nvidia-smi` - # to be available in order to determine the compute capability of the GPU.[^1] - # - # And bindgen_cuda is what candle depends on, to bypass the Compute compatibility here, - # we need to explicitly setting CUDA_COMPUTE_CAP as one of the environment varaible in order to - # build without any actual physical GPUs inside of the GitHub Actions runner.[^2] - # - # About how bindgen_cuda depends on nvidia-smi, check the actual source code here: - # https://github.com/Narsil/bindgen_cuda/blob/a6b0c891be8ebefb55600d46d8a77a358adc3913/src/lib.rs#L477-L500 - # - # Similar information were seen from TEI (text-embedding-inference) documentations - # https://huggingface.co/docs/text-embeddings-inference/en/custom_container - # - # To understand what compute capability is required, visit [CUDA GPU Compute Capability](https://developer.nvidia.com/cuda-gpus) - # - # In short: - # | Compute Capability | GeForce / RTX | - # |---------------------|----------------| - # | 7.5 | GeForce GTX 1650 Ti, NVIDIA TITAN RTX, GeForce RTX 2000 series... | - # | 8.6 | RTX A series, GeForce RTX 3000 series | - # | 8.9 | GeForce RTX 4000 series, RTX A Ada series | - # | 9.0 | GeForce RTX 5000 series, RTX Pro Blackwell series | - # - # Also, the meaning of how the different ranges of numeric values will impact on which compilers, dependencies, to be used, - # TEI (text-embedding-inference) got a Dockerfile specified that well - # https://github.com/huggingface/text-embeddings-inference/blob/6e900afba71821fdf250e380d7da1f5a6e5e7e27/Dockerfile-cuda#L50-L63 - # - # Thanks - # - https://github.com/Narsil/bindgen_cuda/issues/4 - # - https://github.com/Narsil/bindgen_cuda/issues/8 - # - https://github.com/huggingface/candle/issues/1516#issuecomment-1875440701 - # - # [^1]: compilation - Can I compile a cuda program without having a cuda device - Stack Overflow https://stackoverflow.com/a/20196425 - # [^2]: https://huggingface.github.io/candle/guide/installation.html - CUDA_COMPUTE_CAP: '75' # --------- # Nightly (schedule) builds only diff --git a/Cargo.lock b/Cargo.lock index 57fc31cf8..967999d2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,9 +166,6 @@ version = "0.7.0-alpha.1" dependencies = [ "anyhow", "byteorder", - "candle-core", - "candle-nn", - "candle-transformers", "clap", "crossbeam-channel", "hf-hub", @@ -183,7 +180,7 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-global-shortcut", - "tauri-plugin-ipc-audio-transcription-candle", + "tauri-plugin-ipc-audio-transcription-ort", "tauri-plugin-ipc-audio-vad-ort", "tauri-plugin-log", "tauri-plugin-mcp", @@ -199,15 +196,6 @@ dependencies = [ "url", ] -[[package]] -name = "arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "arrayvec" version = "0.7.6" @@ -288,32 +276,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" -[[package]] -name = "bindgen_cuda" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f8489af5b7d17a81bffe37e0f4d6e1e4de87c87329d05447f22c35d95a1227d" -dependencies = [ - "glob", - "num_cpus", - "rayon", -] - -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -341,12 +303,6 @@ dependencies = [ "wyz", ] -[[package]] -name = "block" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" - [[package]] name = "block-buffer" version = "0.10.4" @@ -462,20 +418,6 @@ name = "bytemuck" version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "441473f2b4b0459a68628c744bc61d23e730fb00128b841d30fa4bb3972257e4" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", -] [[package]] name = "byteorder" @@ -532,92 +474,6 @@ dependencies = [ "serde", ] -[[package]] -name = "candle-core" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f51e2ecf6efe9737af8f993433c839f956d2b6ed4fd2dd4a7c6d8b0fa667ff" -dependencies = [ - "byteorder", - "candle-kernels", - "candle-metal-kernels", - "cudarc", - "gemm 0.17.1", - "half", - "memmap2", - "metal 0.27.0", - "num-traits", - "num_cpus", - "rand 0.9.2", - "rand_distr", - "rayon", - "safetensors", - "thiserror 1.0.69", - "ug", - "ug-cuda", - "ug-metal", - "yoke 0.7.5", - "zip", -] - -[[package]] -name = "candle-kernels" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fcd989c2143aa754370b5bfee309e35fbd259e83d9ecf7a73d23d8508430775" -dependencies = [ - "bindgen_cuda", -] - -[[package]] -name = "candle-metal-kernels" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a323ee9c813707f73b6e59300661b354a70410f31fe4135170c4eda8a061534" -dependencies = [ - "half", - "metal 0.27.0", - "once_cell", - "thiserror 1.0.69", - "tracing", -] - -[[package]] -name = "candle-nn" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1980d53280c8f9e2c6cbe1785855d7ff8010208b46e21252b978badf13ad69d" -dependencies = [ - "candle-core", - "candle-metal-kernels", - "half", - "metal 0.27.0", - "num-traits", - "rayon", - "safetensors", - "serde", - "thiserror 1.0.69", -] - -[[package]] -name = "candle-transformers" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186cb80045dbe47e0b387ea6d3e906f02fb3056297080d9922984c90e90a72b0" -dependencies = [ - "byteorder", - "candle-core", - "candle-nn", - "fancy-regex", - "num-traits", - "rand 0.9.2", - "rayon", - "serde", - "serde_json", - "serde_plain", - "tracing", -] - [[package]] name = "cargo-platform" version = "0.1.9" @@ -857,22 +713,11 @@ checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" dependencies = [ "bitflags 2.9.1", "core-foundation 0.10.1", - "core-graphics-types 0.2.0", + "core-graphics-types", "foreign-types 0.5.0", "libc", ] -[[package]] -name = "core-graphics-types" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" -dependencies = [ - "bitflags 1.3.2", - "core-foundation 0.9.4", - "libc", -] - [[package]] name = "core-graphics-types" version = "0.2.0" @@ -936,12 +781,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.6" @@ -989,16 +828,6 @@ dependencies = [ "syn 2.0.104", ] -[[package]] -name = "cudarc" -version = "0.16.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17200eb07e7d85a243aa1bf4569a7aa998385ba98d14833973a817a63cc86e92" -dependencies = [ - "half", - "libloading 0.8.8", -] - [[package]] name = "darling" version = "0.20.11" @@ -1063,17 +892,6 @@ dependencies = [ "serde", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", -] - [[package]] name = "derive_builder" version = "0.20.2" @@ -1235,25 +1053,6 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" -[[package]] -name = "dyn-stack" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" -dependencies = [ - "bytemuck", - "reborrow", -] - -[[package]] -name = "dyn-stack" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490bd48eb68fffcfed519b4edbfd82c69cbe741d175b84f0e0cbe8c57cbe0bdd" -dependencies = [ - "bytemuck", -] - [[package]] name = "either" version = "1.15.0" @@ -1295,18 +1094,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.104", -] - [[package]] name = "env_filter" version = "0.1.3" @@ -1356,17 +1143,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" -[[package]] -name = "fancy-regex" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -1693,243 +1469,6 @@ dependencies = [ "x11", ] -[[package]] -name = "gemm" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-c32 0.17.1", - "gemm-c64 0.17.1", - "gemm-common 0.17.1", - "gemm-f16 0.17.1", - "gemm-f32 0.17.1", - "gemm-f64 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" -dependencies = [ - "dyn-stack 0.13.0", - "gemm-c32 0.18.2", - "gemm-c64 0.18.2", - "gemm-common 0.18.2", - "gemm-f16 0.18.2", - "gemm-f32 0.18.2", - "gemm-f64 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.5.0", - "seq-macro", -] - -[[package]] -name = "gemm-c32" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-c32" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" -dependencies = [ - "dyn-stack 0.13.0", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.5.0", - "seq-macro", -] - -[[package]] -name = "gemm-c64" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-c64" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" -dependencies = [ - "dyn-stack 0.13.0", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.5.0", - "seq-macro", -] - -[[package]] -name = "gemm-common" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" -dependencies = [ - "bytemuck", - "dyn-stack 0.10.0", - "half", - "num-complex", - "num-traits", - "once_cell", - "paste", - "pulp 0.18.22", - "raw-cpuid 10.7.0", - "rayon", - "seq-macro", - "sysctl 0.5.5", -] - -[[package]] -name = "gemm-common" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" -dependencies = [ - "bytemuck", - "dyn-stack 0.13.0", - "half", - "libm", - "num-complex", - "num-traits", - "once_cell", - "paste", - "pulp 0.21.5", - "raw-cpuid 11.5.0", - "rayon", - "seq-macro", - "sysctl 0.6.0", -] - -[[package]] -name = "gemm-f16" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "gemm-f32 0.17.1", - "half", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "rayon", - "seq-macro", -] - -[[package]] -name = "gemm-f16" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" -dependencies = [ - "dyn-stack 0.13.0", - "gemm-common 0.18.2", - "gemm-f32 0.18.2", - "half", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.5.0", - "rayon", - "seq-macro", -] - -[[package]] -name = "gemm-f32" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-f32" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" -dependencies = [ - "dyn-stack 0.13.0", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.5.0", - "seq-macro", -] - -[[package]] -name = "gemm-f64" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" -dependencies = [ - "dyn-stack 0.10.0", - "gemm-common 0.17.1", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 10.7.0", - "seq-macro", -] - -[[package]] -name = "gemm-f64" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" -dependencies = [ - "dyn-stack 0.13.0", - "gemm-common 0.18.2", - "num-complex", - "num-traits", - "paste", - "raw-cpuid 11.5.0", - "seq-macro", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2185,20 +1724,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" -dependencies = [ - "bytemuck", - "cfg-if", - "crunchy", - "num-traits", - "rand 0.9.2", - "rand_distr", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -2434,7 +1959,7 @@ checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", "potential_utf", - "yoke 0.8.0", + "yoke", "zerofrom", "zerovec", ] @@ -2506,7 +2031,7 @@ dependencies = [ "stable_deref_trait", "tinystr", "writeable", - "yoke 0.8.0", + "yoke", "zerofrom", "zerotrie", "zerovec", @@ -2769,7 +2294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9ec52138abedcc58dc17a7c6c0c00a2bdb4f3427c7f63fa97fd0d859155caf" dependencies = [ "gtk-sys", - "libloading 0.7.4", + "libloading", "once_cell", ] @@ -2789,22 +2314,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "libloading" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" -dependencies = [ - "cfg-if", - "windows-targets 0.53.2", -] - -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - [[package]] name = "libredox" version = "0.1.6" @@ -2875,15 +2384,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" -[[package]] -name = "malloc_buf" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" -dependencies = [ - "libc", -] - [[package]] name = "markup5ever" version = "0.14.1" @@ -2931,16 +2431,6 @@ version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" -[[package]] -name = "memmap2" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28" -dependencies = [ - "libc", - "stable_deref_trait", -] - [[package]] name = "memoffset" version = "0.9.1" @@ -2950,36 +2440,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "metal" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43f73953f8cbe511f021b58f18c3ce1c3d1ae13fe953293e13345bf83217f25" -dependencies = [ - "bitflags 2.9.1", - "block", - "core-graphics-types 0.1.3", - "foreign-types 0.5.0", - "log", - "objc", - "paste", -] - -[[package]] -name = "metal" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ecfd3296f8c56b7c1f6fbac3c71cefa9d78ce009850c45000015f206dc7fa21" -dependencies = [ - "bitflags 2.9.1", - "block", - "core-graphics-types 0.1.3", - "foreign-types 0.5.0", - "log", - "objc", - "paste", -] - [[package]] name = "mime" version = "0.3.17" @@ -3139,37 +2599,12 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - [[package]] name = "num-complex" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ - "bytemuck", "num-traits", ] @@ -3188,28 +2623,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -3217,7 +2630,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -3267,16 +2679,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" -[[package]] -name = "objc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" -dependencies = [ - "malloc_buf", - "objc_exception", -] - [[package]] name = "objc-sys" version = "0.3.5" @@ -3491,15 +2893,6 @@ dependencies = [ "objc2-foundation 0.3.1", ] -[[package]] -name = "objc_exception" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" -dependencies = [ - "cc", -] - [[package]] name = "object" version = "0.36.7" @@ -4018,32 +3411,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "pulp" -version = "0.18.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" -dependencies = [ - "bytemuck", - "libm", - "num-complex", - "reborrow", -] - -[[package]] -name = "pulp" -version = "0.21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b86df24f0a7ddd5e4b95c94fc9ed8a98f1ca94d3b01bdce2824097e7835907" -dependencies = [ - "bytemuck", - "cfg-if", - "libm", - "num-complex", - "reborrow", - "version_check", -] - [[package]] name = "quick-xml" version = "0.38.0" @@ -4166,16 +3533,6 @@ dependencies = [ "getrandom 0.3.3", ] -[[package]] -name = "rand_distr" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" -dependencies = [ - "num-traits", - "rand 0.9.2", -] - [[package]] name = "rand_hc" version = "0.2.0" @@ -4194,24 +3551,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "raw-cpuid" -version = "11.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" -dependencies = [ - "bitflags 2.9.1", -] - [[package]] name = "raw-window-handle" version = "0.6.2" @@ -4264,12 +3603,6 @@ dependencies = [ "rustfft", ] -[[package]] -name = "reborrow" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" - [[package]] name = "redox_syscall" version = "0.5.14" @@ -4605,16 +3938,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "safetensors" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "same-file" version = "1.0.6" @@ -4746,12 +4069,6 @@ dependencies = [ "serde", ] -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - [[package]] name = "serde" version = "1.0.219" @@ -4806,15 +4123,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_plain" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" -dependencies = [ - "serde", -] - [[package]] name = "serde_repr" version = "0.1.20" @@ -5384,34 +4692,6 @@ dependencies = [ "libc", ] -[[package]] -name = "sysctl" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" -dependencies = [ - "bitflags 2.9.1", - "byteorder", - "enum-as-inner", - "libc", - "thiserror 1.0.69", - "walkdir", -] - -[[package]] -name = "sysctl" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" -dependencies = [ - "bitflags 2.9.1", - "byteorder", - "enum-as-inner", - "libc", - "thiserror 1.0.69", - "walkdir", -] - [[package]] name = "system-configuration" version = "0.6.1" @@ -5667,19 +4947,18 @@ dependencies = [ ] [[package]] -name = "tauri-plugin-ipc-audio-transcription-candle" +name = "tauri-plugin-ipc-audio-transcription-ort" version = "0.7.0-alpha.1" dependencies = [ "anyhow", "byteorder", - "candle-core", - "candle-nn", - "candle-transformers", "clap", "hf-hub", "log", "ndarray", + "ort", "rubato", + "rustfft", "serde", "serde_json", "symphonia", @@ -6421,54 +5700,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" -[[package]] -name = "ug" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90b70b37e9074642bc5f60bb23247fd072a84314ca9e71cdf8527593406a0dd3" -dependencies = [ - "gemm 0.18.2", - "half", - "libloading 0.8.8", - "memmap2", - "num", - "num-traits", - "num_cpus", - "rayon", - "safetensors", - "serde", - "thiserror 1.0.69", - "tracing", - "yoke 0.7.5", -] - -[[package]] -name = "ug-cuda" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14053653d0b7fa7b21015aa9a62edc8af2f60aa6f9c54e66386ecce55f22ed29" -dependencies = [ - "cudarc", - "half", - "serde", - "thiserror 1.0.69", - "ug", -] - -[[package]] -name = "ug-metal" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76daec3c7a32a1b4a0e3307b6b057fa067aa64e750713987410a2c402e5cd731" -dependencies = [ - "half", - "metal 0.29.0", - "objc", - "serde", - "thiserror 1.0.69", - "ug", -] - [[package]] name = "unic-char-property" version = "0.9.0" @@ -7569,18 +6800,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive 0.7.5", - "zerofrom", -] - [[package]] name = "yoke" version = "0.8.0" @@ -7589,22 +6808,10 @@ checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", - "yoke-derive 0.8.0", + "yoke-derive", "zerofrom", ] -[[package]] -name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", - "synstructure", -] - [[package]] name = "yoke-derive" version = "0.8.0" @@ -7671,7 +6878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" dependencies = [ "displaydoc", - "yoke 0.8.0", + "yoke", "zerofrom", ] @@ -7681,7 +6888,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ - "yoke 0.8.0", + "yoke", "zerofrom", "zerovec-derive", ] @@ -7696,18 +6903,3 @@ dependencies = [ "quote", "syn 2.0.104", ] - -[[package]] -name = "zip" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "indexmap 2.10.0", - "num_enum", - "thiserror 1.0.69", -] diff --git a/Cargo.toml b/Cargo.toml index 64f8ba4f8..bfce543d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = [ - "crates/tauri-plugin-ipc-audio-transcription-candle", + "crates/tauri-plugin-ipc-audio-transcription-ort", "crates/tauri-plugin-ipc-audio-vad-ort", "crates/tauri-plugin-mcp", "crates/tauri-plugin-window-pass-through-on-hover", @@ -13,7 +13,7 @@ resolver = "2" version = "0.7.0-alpha.1" [workspace.dependencies] -tauri-plugin-ipc-audio-transcription-candle = { path = "./crates/tauri-plugin-ipc-audio-transcription-candle" } +tauri-plugin-ipc-audio-transcription-ort = { path = "./crates/tauri-plugin-ipc-audio-transcription-ort" } tauri-plugin-ipc-audio-vad-ort = { path = "./crates/tauri-plugin-ipc-audio-vad-ort" } tauri-plugin-mcp = { path = "./crates/tauri-plugin-mcp" } tauri-plugin-window-pass-through-on-hover = { path = "./crates/tauri-plugin-window-pass-through-on-hover" } diff --git a/apps/stage-tamagotchi/src-tauri/Cargo.toml b/apps/stage-tamagotchi/src-tauri/Cargo.toml index 73b2caa5e..205475512 100644 --- a/apps/stage-tamagotchi/src-tauri/Cargo.toml +++ b/apps/stage-tamagotchi/src-tauri/Cargo.toml @@ -33,7 +33,7 @@ tauri = { version = "2.3.1", features = [ tauri-plugin-log = "2.0.0-rc" tauri-plugin-os = "2" tauri-plugin-mcp = { workspace = true } -tauri-plugin-ipc-audio-transcription-candle = { workspace = true } +tauri-plugin-ipc-audio-transcription-ort = { workspace = true } tauri-plugin-ipc-audio-vad-ort = { workspace = true } tauri-plugin-prevent-default = "1.3" tauri-plugin-window-pass-through-on-hover = { workspace = true } @@ -42,9 +42,6 @@ serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" tokio = "1.45.1" ndarray = "0.16.1" -candle-core = { version = "0.9.1" } -candle-nn = { version = "0.9.1" } -candle-transformers = { version = "0.9.1" } crossbeam-channel = "0.5.15" anyhow = "1.0.98" hf-hub = "0.4.3" diff --git a/apps/stage-tamagotchi/src-tauri/capabilities/default.json b/apps/stage-tamagotchi/src-tauri/capabilities/default.json index e133710e5..ae496499e 100644 --- a/apps/stage-tamagotchi/src-tauri/capabilities/default.json +++ b/apps/stage-tamagotchi/src-tauri/capabilities/default.json @@ -28,7 +28,7 @@ "global-shortcut:allow-unregister-all", "window-pass-through-on-hover:default", "window-router-link:default", - "ipc-audio-transcription-candle:default", + "ipc-audio-transcription-ort:default", "ipc-audio-vad-ort:default" ] } diff --git a/apps/stage-tamagotchi/src-tauri/src/lib.rs b/apps/stage-tamagotchi/src-tauri/src/lib.rs index e6e5b8eb7..da4af1890 100644 --- a/apps/stage-tamagotchi/src-tauri/src/lib.rs +++ b/apps/stage-tamagotchi/src-tauri/src/lib.rs @@ -30,7 +30,7 @@ pub fn run() { .plugin(tauri_plugin_window_state::Builder::default().build()) .plugin(tauri_plugin_positioner::init()) // Project AIRI plugins - .plugin(tauri_plugin_ipc_audio_transcription_candle::init()) + .plugin(tauri_plugin_ipc_audio_transcription_ort::init()) .plugin(tauri_plugin_ipc_audio_vad_ort::init()) .plugin(tauri_plugin_window_pass_through_on_hover::init()) .plugin(tauri_plugin_window_router_link::init( diff --git a/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts b/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts index c4b93d7f6..b4a89c629 100644 --- a/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts +++ b/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts @@ -64,7 +64,7 @@ export function useWhisperTranscription(config: Partial = {}) { error.value = '' try { - await invoke('plugin:ipc-audio-transcription-candle|load_candle_model_whisper', { modelType }) + await invoke('plugin:ipc-audio-transcription-ort|load_ort_model_whisper', { modelType }) isModelLoaded.value = true } catch (err) { @@ -86,16 +86,15 @@ export function useWhisperTranscription(config: Partial = {}) { isProcessing.value = true const audioArray = Array.from(segment.audioData) - const [result, language] = await invoke('plugin:ipc-audio-transcription-candle|ipc_audio_transcription', { + const result = await invoke('plugin:ipc-audio-transcription-ort|ipc_audio_transcription', { chunk: audioArray, language: mapLanguageCodeToName(locale), - }) || ['', ''] + }) || '' const transcription: TranscriptionResult = { id: `transcription_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, segmentId: segment.id, text: result?.trim() || '', - language: language?.trim() || '', processingTimeMs: performance.now() - startTime, timestamp: Date.now(), } diff --git a/apps/stage-tamagotchi/src/composables/tauri.ts b/apps/stage-tamagotchi/src/composables/tauri.ts index aabdedffe..543604038 100644 --- a/apps/stage-tamagotchi/src/composables/tauri.ts +++ b/apps/stage-tamagotchi/src/composables/tauri.ts @@ -54,7 +54,7 @@ export interface AiriTamagotchiEvents extends Events { 'tauri-plugins:tauri-plugin-window-pass-through-on-hover:cursor-position': Point 'tauri-plugins:tauri-plugin-window-pass-through-on-hover:window-frame': WindowFrame 'tauri-plugins:tauri-plugin-window-pass-through-on-hover:pass-through-enabled': boolean - 'tauri-plugins:tauri-plugin-ipc-audio-transcription-candle:load-model-silero-vad-progress': [boolean, string, number, number, number] + 'tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-silero-vad-progress': [boolean, string, number, number, number] 'tauri-plugins:tauri-plugin-ipc-audio-vad-ort:load-model-whisper-progress': [boolean, string, number, number, number] 'mcp_plugin_destroyed': undefined } diff --git a/apps/stage-tamagotchi/src/pages/index.vue b/apps/stage-tamagotchi/src/pages/index.vue index 54c9d6aee..1039accd8 100644 --- a/apps/stage-tamagotchi/src/pages/index.vue +++ b/apps/stage-tamagotchi/src/pages/index.vue @@ -71,7 +71,7 @@ function openChat() { onMounted(async () => { // VAD - unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-ipc-audio-transcription-candle:load-model-silero-vad-progress', (event) => { + unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-silero-vad-progress', (event) => { const [_, filename, progress, totalSize, currentSize] = event.payload resourcesStore.updateResourceProgress('hearing', 'vad', { filename, progress, totalSize, currentSize }) })) @@ -82,7 +82,7 @@ onMounted(async () => { const [_, filename, progress, totalSize, currentSize] = event.payload resourcesStore.updateResourceProgress('hearing', 'whisper', { filename, progress, totalSize, currentSize }) })) - invoke('plugin:ipc-audio-transcription-candle|load_candle_model_whisper', { modelType: 'medium' }) + invoke('plugin:ipc-audio-transcription-ort|load_ort_model_whisper', { modelType: 'medium' }) if (connected.value) return diff --git a/apps/stage-tamagotchi/src/tauri/invoke.ts b/apps/stage-tamagotchi/src/tauri/invoke.ts index 2f4a340cd..7205f2d21 100644 --- a/apps/stage-tamagotchi/src/tauri/invoke.ts +++ b/apps/stage-tamagotchi/src/tauri/invoke.ts @@ -8,8 +8,8 @@ export interface InvokeMethods { 'open_chat_window': { args: undefined, options: undefined, returns: void } // Plugin - Audio Transcription - 'plugin:ipc-audio-transcription-candle|load_candle_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void } - 'plugin:ipc-audio-transcription-candle|ipc_audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: [string, string] } + 'plugin:ipc-audio-transcription-ort|load_ort_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void } + 'plugin:ipc-audio-transcription-ort|ipc_audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: string } // Plugin - Audio VAD 'plugin:ipc-audio-vad-ort|load_ort_model_silero_vad': { args: undefined, options: undefined, returns: void } diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/Cargo.toml b/crates/tauri-plugin-ipc-audio-transcription-candle/Cargo.toml deleted file mode 100644 index 4111af445..000000000 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/Cargo.toml +++ /dev/null @@ -1,62 +0,0 @@ -[package] -name = "tauri-plugin-ipc-audio-transcription-candle" -version.workspace = true -description = "Audio transcription plugin for Tauri using Candle (huggingface/candle)" -authors = [ "Neko Ayaka " ] -license = "MIT" -repository = "https://github.com/moeru-ai/airi" -edition = "2024" -rust-version = "1.85.0" -links = "tauri-plugin-ipc-audio-transcription-candle" - -[lib] -name = "tauri_plugin_ipc_audio_transcription_candle" -crate-type = [ - "staticlib", - "cdylib", - "rlib" -] - -[features] -default = [] -cuda = [ - "candle-core/cuda", - "candle-nn/cuda", - "candle-transformers/cuda" -] - -[dependencies] -tauri = "2.3.1" -serde = { version = "1.0.219", features = ["derive"] } -serde_json = "1.0.140" -tokio = "1.45.1" -ndarray = "0.16.1" -candle-core = { version = "0.9.1" } -candle-nn = { version = "0.9.1" } -candle-transformers = { version = "0.9.1" } -log = "0.4" -anyhow = "1" -tokenizers = "0.21.2" -hf-hub = "0.4.3" -symphonia = "0.5.4" -rubato = "0.16.2" -byteorder = "1.5.0" -clap = { version = "4.5.40", features = ["derive"] } - -[target.'cfg(target_os = "macos")'.dependencies] -candle-core = { version = "0.9.1", features = ["metal"] } -candle-nn = { version = "0.9.1", features = ["metal"] } -candle-transformers = { version = "0.9.1", features = ["metal"] } - -[target.'cfg(target_os = "windows")'.dependencies] -candle-core = { version = "0.9.1" } -candle-nn = { version = "0.9.1" } -candle-transformers = { version = "0.9.1" } - -[target.'cfg(target_os = "linux")'.dependencies] -candle-core = { version = "0.9.1" } -candle-nn = { version = "0.9.1" } -candle-transformers = { version = "0.9.1" } - -[build-dependencies] -tauri-plugin = { version = "2.2", features = [ "build" ] } diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/build.rs b/crates/tauri-plugin-ipc-audio-transcription-candle/build.rs deleted file mode 100644 index 647cdd9d9..000000000 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/build.rs +++ /dev/null @@ -1,5 +0,0 @@ -const COMMANDS: &[&str] = &["load_candle_model_whisper", "ipc_audio_transcription"]; - -fn main() { - tauri_plugin::Builder::new(COMMANDS).build(); -} diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/mod.rs b/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/mod.rs deleted file mode 100644 index 75c349281..000000000 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -pub mod whisper; - -use log::info; -use tauri::Runtime; - -use crate::{helpers::huggingface::load_device, models::whisper::WhichWhisperModel}; - -pub fn new_whisper_processor( - window: tauri::WebviewWindow, - model_type: Option, -) -> anyhow::Result { - let device = load_device().map_err(|err| anyhow::anyhow!("Failed to load device: {}", err))?; - let whisper_model = model_type.unwrap_or_else(|| { - if device.is_cuda() { - WhichWhisperModel::LargeV3 - } else if device.is_metal() { - WhichWhisperModel::Base - } else { - WhichWhisperModel::Tiny - } - }); - info!("Loading whisper model: {:?}", whisper_model); - whisper::Processor::new(whisper_model, device.clone(), window) -} diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/languages.rs b/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/languages.rs deleted file mode 100644 index 42f41370d..000000000 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/languages.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Language constants -pub const LANGUAGES: [(&str, &str); 99] = [ - ("en", "english"), - ("zh", "chinese"), - ("de", "german"), - ("es", "spanish"), - ("ru", "russian"), - ("ko", "korean"), - ("fr", "french"), - ("ja", "japanese"), - ("pt", "portuguese"), - ("tr", "turkish"), - ("pl", "polish"), - ("ca", "catalan"), - ("nl", "dutch"), - ("ar", "arabic"), - ("sv", "swedish"), - ("it", "italian"), - ("id", "indonesian"), - ("hi", "hindi"), - ("fi", "finnish"), - ("vi", "vietnamese"), - ("he", "hebrew"), - ("uk", "ukrainian"), - ("el", "greek"), - ("ms", "malay"), - ("cs", "czech"), - ("ro", "romanian"), - ("da", "danish"), - ("hu", "hungarian"), - ("ta", "tamil"), - ("no", "norwegian"), - ("th", "thai"), - ("ur", "urdu"), - ("hr", "croatian"), - ("bg", "bulgarian"), - ("lt", "lithuanian"), - ("la", "latin"), - ("mi", "maori"), - ("ml", "malayalam"), - ("cy", "welsh"), - ("sk", "slovak"), - ("te", "telugu"), - ("fa", "persian"), - ("lv", "latvian"), - ("bn", "bengali"), - ("sr", "serbian"), - ("az", "azerbaijani"), - ("sl", "slovenian"), - ("kn", "kannada"), - ("et", "estonian"), - ("mk", "macedonian"), - ("br", "breton"), - ("eu", "basque"), - ("is", "icelandic"), - ("hy", "armenian"), - ("ne", "nepali"), - ("mn", "mongolian"), - ("bs", "bosnian"), - ("kk", "kazakh"), - ("sq", "albanian"), - ("sw", "swahili"), - ("gl", "galician"), - ("mr", "marathi"), - ("pa", "punjabi"), - ("si", "sinhala"), - ("km", "khmer"), - ("sn", "shona"), - ("yo", "yoruba"), - ("so", "somali"), - ("af", "afrikaans"), - ("oc", "occitan"), - ("ka", "georgian"), - ("be", "belarusian"), - ("tg", "tajik"), - ("sd", "sindhi"), - ("gu", "gujarati"), - ("am", "amharic"), - ("yi", "yiddish"), - ("lo", "lao"), - ("uz", "uzbek"), - ("fo", "faroese"), - ("ht", "haitian creole"), - ("ps", "pashto"), - ("tk", "turkmen"), - ("nn", "nynorsk"), - ("mt", "maltese"), - ("sa", "sanskrit"), - ("lb", "luxembourgish"), - ("my", "myanmar"), - ("bo", "tibetan"), - ("tl", "tagalog"), - ("mg", "malagasy"), - ("as", "assamese"), - ("tt", "tatar"), - ("haw", "hawaiian"), - ("ln", "lingala"), - ("ha", "hausa"), - ("ba", "bashkir"), - ("jw", "javanese"), - ("su", "sundanese"), -]; diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/mod.rs b/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/mod.rs deleted file mode 100644 index 1d7421154..000000000 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/mod.rs +++ /dev/null @@ -1,478 +0,0 @@ -use anyhow::Result; -use byteorder::{ByteOrder, LittleEndian}; -use candle_core::{D, Device, IndexOp, Tensor}; -use candle_nn::{VarBuilder, ops::softmax}; -use candle_transformers::models::whisper::{self as whisper_model, Config, audio}; -use clap::ValueEnum; -use hf_hub::{Repo, RepoType, api::sync::ApiBuilder}; -use log::info; -use tauri::Runtime; -use tokenizers::Tokenizer; - -use crate::helpers::huggingface::create_progress_emitter; - -mod languages; - -pub enum WhisperModel { - Normal(whisper_model::model::Whisper), -} - -impl WhisperModel { - pub fn encoder_forward( - &mut self, - x: &Tensor, - flush: bool, - ) -> candle_core::Result { - match self { - Self::Normal(model) => model.encoder.forward(x, flush), - } - } - - pub fn decoder_forward( - &mut self, - x: &Tensor, - encoder_out: &Tensor, - flush: bool, - ) -> candle_core::Result { - match self { - Self::Normal(model) => model.decoder.forward(x, encoder_out, flush), - } - } - - pub fn decoder_final_linear( - &self, - x: &Tensor, - ) -> candle_core::Result { - match self { - Self::Normal(model) => model.decoder.final_linear(x), - } - } - - pub fn reset_kv_cache(&mut self) { - match self { - Self::Normal(model) => model.reset_kv_cache(), - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum WhichWhisperModel { - Tiny, - #[value(name = "tiny.en")] - TinyEn, - Base, - #[value(name = "base.en")] - BaseEn, - Small, - #[value(name = "small.en")] - SmallEn, - Medium, - #[value(name = "medium.en")] - MediumEn, - Large, - LargeV2, - LargeV3, - LargeV3Turbo, - #[value(name = "distil-medium.en")] - DistilMediumEn, - #[value(name = "distil-large-v2")] - DistilLargeV2, -} - -impl WhichWhisperModel { - pub const fn model_and_revision(self) -> (&'static str, &'static str) { - match self { - Self::Tiny => ("openai/whisper-tiny", "main"), - Self::TinyEn => ("openai/whisper-tiny.en", "refs/pr/15"), - Self::Base => ("openai/whisper-base", "refs/pr/22"), - Self::BaseEn => ("openai/whisper-base.en", "refs/pr/13"), - Self::Small => ("openai/whisper-small", "main"), - Self::SmallEn => ("openai/whisper-small.en", "refs/pr/10"), - Self::Medium => ("openai/whisper-medium", "main"), - Self::MediumEn => ("openai/whisper-medium.en", "main"), - Self::Large => ("openai/whisper-large", "refs/pr/36"), - Self::LargeV2 => ("openai/whisper-large-v2", "refs/pr/57"), - Self::LargeV3 => ("openai/whisper-large-v3", "main"), - Self::LargeV3Turbo => ("openai/whisper-large-v3-turbo", "main"), - Self::DistilMediumEn => ("distil-whisper/distil-medium.en", "main"), - Self::DistilLargeV2 => ("distil-whisper/distil-large-v2", "main"), - } - } - - pub const fn is_multilingual(self) -> bool { - match self { - Self::Tiny - | Self::Base - | Self::Small - | Self::Medium - | Self::Large - | Self::LargeV2 - | Self::LargeV3 - | Self::LargeV3Turbo - | Self::DistilLargeV2 => true, - Self::TinyEn | Self::BaseEn | Self::SmallEn | Self::MediumEn | Self::DistilMediumEn => false, - } - } -} - -pub struct Processor { - pub model: WhisperModel, - pub tokenizer: Tokenizer, - pub config: Config, - pub mel_filters: Vec, - pub device: Device, - pub model_type: WhichWhisperModel, - - // Special tokens - pub sot_token: u32, - pub transcribe_token: u32, - pub translate_token: u32, - pub eot_token: u32, - pub no_timestamps_token: u32, - pub suppress_tokens: Tensor, -} - -impl Processor { - pub fn new( - model_type: WhichWhisperModel, - device: Device, - window: tauri::WebviewWindow, - ) -> Result { - let (model_id, revision) = model_type.model_and_revision(); - - let cache_api = hf_hub::Cache::from_env(); - let cache_repo = cache_api.repo(Repo::with_revision( - model_id.to_string(), - RepoType::Model, - revision.to_string(), - )); - - let api = ApiBuilder::new().with_progress(false).build()?; - let repo = api.repo(Repo::with_revision( - model_id.to_string(), - RepoType::Model, - revision.to_string(), - )); - - let config_filename_sub_name = "config.json"; - let config_filename = match cache_repo.get(config_filename_sub_name) { - None => repo.download_with_progress( - config_filename_sub_name, - create_progress_emitter( - window.clone(), - "tauri-plugins:tauri-plugin-ipc-audio-transcription-candle:load-model-whisper-progress", - config_filename_sub_name.to_string(), - ), - )?, - Some(p) => p, - }; - info!("config_filename: {:?}", config_filename.display()); - - let tokenizer_filename_sub_name = "tokenizer.json"; - let tokenizer_filename = match cache_repo.get(tokenizer_filename_sub_name) { - None => repo.download_with_progress( - tokenizer_filename_sub_name, - create_progress_emitter( - window.clone(), - "tauri-plugins:tauri-plugin-ipc-audio-transcription-candle:load-model-whisper-progress", - tokenizer_filename_sub_name.to_string(), - ), - )?, - Some(p) => p, - }; - info!("tokenizer_filename: {:?}", tokenizer_filename.display()); - - let model_filename_sub_name = "model.safetensors"; - let model_filename = match cache_repo.get(model_filename_sub_name) { - None => repo.download_with_progress( - model_filename_sub_name, - create_progress_emitter( - window.clone(), - "tauri-plugins:tauri-plugin-ipc-audio-transcription-candle:load-model-whisper-progress", - model_filename_sub_name.to_string(), - ), - )?, - Some(p) => p, - }; - info!("model_filename: {:?}", model_filename.display()); - - let config: Config = serde_json::from_str(&std::fs::read_to_string(config_filename)?)?; - let tokenizer = Tokenizer::from_file(tokenizer_filename).map_err(anyhow::Error::msg)?; - - info!("Loading Whisper model from: {:?}", model_filename.display()); - - // SAFETY: This is safe because we are using a mmaped file and the safetensors library guarantees that the data is valid. - let var_builder = unsafe { - VarBuilder::from_mmaped_safetensors(&[model_filename], whisper_model::DTYPE, &device)? - }; - - let model = WhisperModel::Normal(whisper_model::model::Whisper::load( - &var_builder, - config.clone(), - )?); - - let mel_bytes = match config.num_mel_bins { - 80 => include_bytes!("./melfilters.bytes").as_slice(), - 128 => include_bytes!("./melfilters128.bytes").as_slice(), - num_mel_bins => anyhow::bail!("Unsupported number of mel bins: {}", num_mel_bins), - }; - - let mut mel_filters = vec![0f32; mel_bytes.len() / 4]; - ::read_f32_into(mel_bytes, &mut mel_filters); - - // Initialize special tokens - let sot_token = Self::token_id_static(&tokenizer, whisper_model::SOT_TOKEN)?; - let transcribe_token = Self::token_id_static(&tokenizer, whisper_model::TRANSCRIBE_TOKEN)?; - let translate_token = Self::token_id_static(&tokenizer, whisper_model::TRANSLATE_TOKEN)?; - let eot_token = Self::token_id_static(&tokenizer, whisper_model::EOT_TOKEN)?; - let no_timestamps_token = - Self::token_id_static(&tokenizer, whisper_model::NO_TIMESTAMPS_TOKEN)?; - - // Create suppress tokens - let suppress_values: Vec = (0..config.vocab_size as u32) - .map(|i| { - if config.suppress_tokens.contains(&i) { - f32::NEG_INFINITY - } else { - 0.0f32 - } - }) - .collect(); - - let suppress_tokens = Tensor::new(suppress_values.as_slice(), &device)?; - - Ok(Self { - model, - tokenizer, - config, - mel_filters, - device, - model_type, - sot_token, - transcribe_token, - translate_token, - eot_token, - no_timestamps_token, - suppress_tokens, - }) - } - - pub fn transcribe( - &mut self, - audio: &[f32], - language: Option<&str>, - ) -> Result<(String, String)> { - // Convert PCM to mel spectrogram - let mel = audio::pcm_to_mel(&self.config, audio, &self.mel_filters); - let mel_len = mel.len(); - let mel = Tensor::from_vec( - mel, - ( - 1, - self.config.num_mel_bins, - mel_len / self.config.num_mel_bins, - ), - &self.device, - )?; - - // Get language token - either specified or detected - let (language_token, detected_language) = match language { - Some(lang) => { - let token = self.get_language_token(lang)?; - (Some(token), lang.to_string()) - }, - None => { - if self.model_type.is_multilingual() { - let (token, lang) = self.detect_language(&mel)?; - (Some(token), lang) - } else { - // English-only models don't use language tokens - (None, "english".to_string()) - } - }, - }; - - // Run encoder - let audio_features = self.model.encoder_forward(&mel, true)?; - - // Decode with the determined language - let tokens = self.decode_whisper_with_language_token(&audio_features, language_token)?; - - // Filter out special tokens - let filtered_tokens: Vec = tokens - .into_iter() - .filter(|&token| !self.is_special_token(token)) - .collect(); - - let text = self - .tokenizer - .decode(&filtered_tokens, true) - .map_err(anyhow::Error::msg)?; - - // Reset KV cache for next inference - self.model.reset_kv_cache(); - - Ok((text.trim().to_string(), detected_language)) - } - - fn detect_language( - &mut self, - mel: &Tensor, - ) -> Result<(u32, String)> { - let (_bsize, _, seq_len) = mel.dims3()?; - let mel = mel.narrow(2, 0, usize::min(seq_len, self.config.max_source_positions))?; - let device = mel.device(); - - // Get all language token IDs - let language_token_ids = languages::LANGUAGES - .iter() - .map(|(code, _)| Self::token_id_static(&self.tokenizer, &format!("<|{}|>", code))) - .collect::, _>>()?; - - let audio_features = self.model.encoder_forward(&mel, true)?; - let tokens = Tensor::new(&[[self.sot_token]], device)?; - let language_token_ids_tensor = Tensor::new(language_token_ids.as_slice(), device)?; - - let ys = self - .model - .decoder_forward(&tokens, &audio_features, true)?; - let logits = self - .model - .decoder_final_linear(&ys.i(..1)?)? - .i(0)? - .i(0)?; - - // Get logits for language tokens only - let logits = logits.index_select(&language_token_ids_tensor, 0)?; - let probs = softmax(&logits, D::Minus1)?; - let probs = probs.to_vec1::()?; - - // Combine with language info and sort by probability - let mut language_probs = languages::LANGUAGES - .iter() - .zip(probs.iter()) - .collect::>(); - - language_probs.sort_by(|(_, p1), (_, p2)| p2.total_cmp(p1)); - - // Log top 5 detected languages - info!("Language detection results:"); - for ((_, name), prob) in language_probs.iter().take(5) { - info!(" {}: {:.3}", name, prob); - } - - // Return the most likely language - let best_language = language_probs[0].0; - let language_token = - Self::token_id_static(&self.tokenizer, &format!("<|{}|>", best_language.0))?; - - Ok((language_token, best_language.1.to_string())) - } - - fn get_language_token( - &self, - language: &str, - ) -> Result { - let language_lower = language.to_lowercase(); - - // Check if it's a language code or name - for (code, name) in languages::LANGUAGES.iter() { - if code == &language_lower || name == &language_lower { - let token = format!("<|{}|>", code); - return Self::token_id_static(&self.tokenizer, &token); - } - } - - anyhow::bail!("Unsupported language: {}", language) - } - - fn decode_whisper_with_language_token( - &mut self, - audio_features: &Tensor, - language_token: Option, - ) -> Result> { - // Initialize with proper Whisper token sequence - let mut tokens = vec![self.sot_token]; - - // Add language token if provided (for multilingual models) - if let Some(lang_token) = language_token { - tokens.push(lang_token); - } - - tokens.push(self.transcribe_token); - tokens.push(self.no_timestamps_token); - - let max_len = 200; - - for i in 0..max_len { - let tokens_t = Tensor::new(tokens.as_slice(), &self.device)?.unsqueeze(0)?; - let ys = self - .model - .decoder_forward(&tokens_t, audio_features, i == 0)?; - - let (_, seq_len, _) = ys.dims3()?; - let logits = self - .model - .decoder_final_linear(&ys.i((..1, seq_len - 1..))?)? - .i(0)? - .i(0)?; - - // Apply suppression tokens - let logits = logits.broadcast_add(&self.suppress_tokens)?; - - // Apply softmax and get most likely token - let probs = softmax(&logits, 0)?; - let probs_vec: Vec = probs.to_vec1()?; - - let next_token = probs_vec - .iter() - .enumerate() - .max_by(|(_, a), (_, b)| a.total_cmp(b)) - .map(|(i, _)| i as u32) - .unwrap(); - - tokens.push(next_token); - - if next_token == self.eot_token { - break; - } - } - - Ok(tokens) - } - - fn is_special_token( - &self, - token: u32, - ) -> bool { - // Check basic special tokens - if token == self.sot_token - || token == self.eot_token - || token == self.transcribe_token - || token == self.translate_token - || token == self.no_timestamps_token - { - return true; - } - - // Check if it's a language token - for (code, _) in languages::LANGUAGES.iter() { - if let Ok(lang_token) = Self::token_id_static(&self.tokenizer, &format!("<|{}|>", code)) { - if token == lang_token { - return true; - } - } - } - - false - } - - fn token_id_static( - tokenizer: &Tokenizer, - token: &str, - ) -> Result { - tokenizer - .token_to_id(token) - .ok_or_else(|| anyhow::anyhow!("Token not found: {}", token)) - } -} diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/Cargo.lock b/crates/tauri-plugin-ipc-audio-transcription-ort/Cargo.lock similarity index 100% rename from crates/tauri-plugin-ipc-audio-transcription-candle/Cargo.lock rename to crates/tauri-plugin-ipc-audio-transcription-ort/Cargo.lock diff --git a/crates/tauri-plugin-ipc-audio-transcription-ort/Cargo.toml b/crates/tauri-plugin-ipc-audio-transcription-ort/Cargo.toml new file mode 100644 index 000000000..1dd7dcbca --- /dev/null +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "tauri-plugin-ipc-audio-transcription-ort" +version.workspace = true +description = "Audio transcription plugin for Tauri using ONNX Runtime" +authors = [ "Neko Ayaka " ] +license = "MIT" +repository = "https://github.com/moeru-ai/airi" +edition = "2024" +rust-version = "1.85.0" +links = "tauri-plugin-ipc-audio-transcription-ort" + +[lib] +name = "tauri_plugin_ipc_audio_transcription_ort" +crate-type = [ + "staticlib", + "cdylib", + "rlib" +] + +[features] +default = [] + +[dependencies] +tauri = "2.3.1" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = "1.0.140" +tokio = "1.45.1" +ndarray = "0.16.1" +log = "0.4" +anyhow = "1" +tokenizers = "0.21.2" +hf-hub = "0.4.3" +symphonia = "0.5.4" +rubato = "0.16.2" +byteorder = "1.5.0" +clap = { version = "4.5.40", features = ["derive"] } +rustfft = "6.4.0" + +[target.'cfg(target_os = "macos")'.dependencies] +ort = { version = "2.0.0-rc.10", features = ["ndarray", "coreml"] } + +[target.'cfg(target_os = "windows")'.dependencies] +ort = { version = "2.0.0-rc.10", features = ["ndarray", "directml", "cuda"] } + +[target.'cfg(target_os = "linux")'.dependencies] +ort = { version = "2.0.0-rc.10", features = ["ndarray", "cuda"] } + +[build-dependencies] +tauri-plugin = { version = "2.2", features = [ "build" ] } diff --git a/crates/tauri-plugin-ipc-audio-transcription-ort/build.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/build.rs new file mode 100644 index 000000000..88dbf5528 --- /dev/null +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/build.rs @@ -0,0 +1,5 @@ +const COMMANDS: &[&str] = &["load_ort_model_whisper", "ipc_audio_transcription"]; + +fn main() { + tauri_plugin::Builder::new(COMMANDS).build(); +} diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/autogenerated/commands/ipc_audio_transcription.toml b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/ipc_audio_transcription.toml similarity index 100% rename from crates/tauri-plugin-ipc-audio-transcription-candle/permissions/autogenerated/commands/ipc_audio_transcription.toml rename to crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/ipc_audio_transcription.toml diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/autogenerated/commands/load_candle_model_whisper.toml b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/load_candle_model_whisper.toml similarity index 100% rename from crates/tauri-plugin-ipc-audio-transcription-candle/permissions/autogenerated/commands/load_candle_model_whisper.toml rename to crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/load_candle_model_whisper.toml diff --git a/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/load_ort_model_whisper.toml b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/load_ort_model_whisper.toml new file mode 100644 index 000000000..60a51b5c2 --- /dev/null +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/commands/load_ort_model_whisper.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-load-ort-model-whisper" +description = "Enables the load_ort_model_whisper command without any pre-configured scope." +commands.allow = ["load_ort_model_whisper"] + +[[permission]] +identifier = "deny-load-ort-model-whisper" +description = "Denies the load_ort_model_whisper command without any pre-configured scope." +commands.deny = ["load_ort_model_whisper"] diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/autogenerated/reference.md b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/reference.md similarity index 57% rename from crates/tauri-plugin-ipc-audio-transcription-candle/permissions/autogenerated/reference.md rename to crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/reference.md index d4f9d4d0c..2a23e5076 100644 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/autogenerated/reference.md +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/autogenerated/reference.md @@ -9,7 +9,7 @@ All operations are enabled by default. #### This default permission set includes the following: -- `allow-load-candle-model-whisper` +- `allow-load-ort-model-whisper` - `allow-ipc-audio-transcription` ## Permission Table @@ -24,7 +24,7 @@ All operations are enabled by default. -`ipc-audio-transcription-candle:allow-ipc-audio-transcription` +`ipc-audio-transcription-ort:allow-ipc-audio-transcription` @@ -37,7 +37,7 @@ Enables the ipc_audio_transcription command without any pre-configured scope. -`ipc-audio-transcription-candle:deny-ipc-audio-transcription` +`ipc-audio-transcription-ort:deny-ipc-audio-transcription` @@ -50,7 +50,7 @@ Denies the ipc_audio_transcription command without any pre-configured scope. -`ipc-audio-transcription-candle:allow-load-candle-model-whisper` +`ipc-audio-transcription-ort:allow-load-candle-model-whisper` @@ -63,13 +63,39 @@ Enables the load_candle_model_whisper command without any pre-configured scope. -`ipc-audio-transcription-candle:deny-load-candle-model-whisper` +`ipc-audio-transcription-ort:deny-load-candle-model-whisper` Denies the load_candle_model_whisper command without any pre-configured scope. + + + + + + +`ipc-audio-transcription-ort:allow-load-ort-model-whisper` + + + + +Enables the load_ort_model_whisper command without any pre-configured scope. + + + + + + + +`ipc-audio-transcription-ort:deny-load-ort-model-whisper` + + + + +Denies the load_ort_model_whisper command without any pre-configured scope. + diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/default.toml b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/default.toml similarity index 88% rename from crates/tauri-plugin-ipc-audio-transcription-candle/permissions/default.toml rename to crates/tauri-plugin-ipc-audio-transcription-ort/permissions/default.toml index f6db87deb..a5d6b4b68 100644 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/default.toml +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/default.toml @@ -10,6 +10,6 @@ operations are available from the mcp plugin. All operations are enabled by default. """ permissions = [ - "allow-load-candle-model-whisper", + "allow-load-ort-model-whisper", "allow-ipc-audio-transcription", ] diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/schemas/schema.json b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/schemas/schema.json similarity index 92% rename from crates/tauri-plugin-ipc-audio-transcription-candle/permissions/schemas/schema.json rename to crates/tauri-plugin-ipc-audio-transcription-ort/permissions/schemas/schema.json index ff0a2f131..3195530c5 100644 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/permissions/schemas/schema.json +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/permissions/schemas/schema.json @@ -319,10 +319,22 @@ "markdownDescription": "Denies the load_candle_model_whisper command without any pre-configured scope." }, { - "description": "This permission set configures what kind of\noperations are available from the mcp plugin.\n\n#### Granted Permissions\n\nAll operations are enabled by default.\n\n#### This default permission set includes:\n\n- `allow-load-candle-model-whisper`\n- `allow-ipc-audio-transcription`", + "description": "Enables the load_ort_model_whisper command without any pre-configured scope.", + "type": "string", + "const": "allow-load-ort-model-whisper", + "markdownDescription": "Enables the load_ort_model_whisper command without any pre-configured scope." + }, + { + "description": "Denies the load_ort_model_whisper command without any pre-configured scope.", + "type": "string", + "const": "deny-load-ort-model-whisper", + "markdownDescription": "Denies the load_ort_model_whisper command without any pre-configured scope." + }, + { + "description": "This permission set configures what kind of\noperations are available from the mcp plugin.\n\n#### Granted Permissions\n\nAll operations are enabled by default.\n\n#### This default permission set includes:\n\n- `allow-load-ort-model-whisper`\n- `allow-ipc-audio-transcription`", "type": "string", "const": "default", - "markdownDescription": "This permission set configures what kind of\noperations are available from the mcp plugin.\n\n#### Granted Permissions\n\nAll operations are enabled by default.\n\n#### This default permission set includes:\n\n- `allow-load-candle-model-whisper`\n- `allow-ipc-audio-transcription`" + "markdownDescription": "This permission set configures what kind of\noperations are available from the mcp plugin.\n\n#### Granted Permissions\n\nAll operations are enabled by default.\n\n#### This default permission set includes:\n\n- `allow-load-ort-model-whisper`\n- `allow-ipc-audio-transcription`" } ] } diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/helpers/huggingface.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/src/helpers/huggingface.rs similarity index 86% rename from crates/tauri-plugin-ipc-audio-transcription-candle/src/helpers/huggingface.rs rename to crates/tauri-plugin-ipc-audio-transcription-ort/src/helpers/huggingface.rs index 5eb01c2a4..729ce5a02 100644 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/src/helpers/huggingface.rs +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/src/helpers/huggingface.rs @@ -1,21 +1,6 @@ -use anyhow::Ok; -use log::{error, info}; +use log::error; use tauri::{Emitter, Runtime}; -pub fn load_device() -> anyhow::Result { - // Determine device to use - let device = if candle_core::utils::cuda_is_available() { - candle_core::Device::new_cuda(0)? - } else if candle_core::utils::metal_is_available() { - candle_core::Device::new_metal(0)? - } else { - candle_core::Device::Cpu - }; - - info!("Using device: {device:?}"); - Ok(device) -} - pub trait ProgressEmitter: Send + Sync { fn emit_progress( &self, diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/helpers/mod.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/src/helpers/mod.rs similarity index 100% rename from crates/tauri-plugin-ipc-audio-transcription-candle/src/helpers/mod.rs rename to crates/tauri-plugin-ipc-audio-transcription-ort/src/helpers/mod.rs diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/lib.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/src/lib.rs similarity index 80% rename from crates/tauri-plugin-ipc-audio-transcription-candle/src/lib.rs rename to crates/tauri-plugin-ipc-audio-transcription-ort/src/lib.rs index 9917cd76b..7eeaea13e 100644 --- a/crates/tauri-plugin-ipc-audio-transcription-candle/src/lib.rs +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/src/lib.rs @@ -13,13 +13,16 @@ mod models; #[derive(Default)] struct AppDataWhisperProcessor { - whisper_processor: Option, + whisper_processor: Option, } -use crate::models::{new_whisper_processor, whisper::WhichWhisperModel}; +use crate::models::{ + new_whisper_processor, + whisper::{self, whisper::WhichModel}, +}; #[tauri::command] -async fn load_candle_model_whisper( +async fn load_ort_model_whisper( app: tauri::AppHandle, window: tauri::WebviewWindow, model_type: Option, @@ -38,7 +41,7 @@ async fn load_candle_model_whisper( // Load the traditional whisper models first match new_whisper_processor( window, - Some(WhichWhisperModel::from_str( + Some(WhichModel::from_str( model_type .unwrap_or_else(|| "medium".to_string()) .as_str(), @@ -67,7 +70,7 @@ async fn ipc_audio_transcription( app: tauri::AppHandle, chunk: Vec, language: Option, -) -> Result<(String, String), String> { +) -> Result { info!("Processing audio transcription..."); let data = app.state::>(); @@ -84,24 +87,27 @@ async fn ipc_audio_transcription( let mut data = data.lock().unwrap(); let processor = data.whisper_processor.as_mut().unwrap(); - let (transcription, language) = processor - .transcribe(chunk.as_slice(), language.as_deref()) + let mut config = whisper::whisper::GenerationConfig::default(); + config.language = language; + + let transcription = processor + .transcribe(chunk.as_slice(), &config) .map_err(|e| e.to_string())?; info!("Transcription completed: {}", transcription); - Ok((transcription, language)) + Ok(transcription) } pub fn init() -> TauriPlugin { - PluginBuilder::new("ipc-audio-transcription-candle") + PluginBuilder::new("ipc-audio-transcription-ort") .setup(|app, _| { info!("Initializing audio transcription plugin..."); app.manage(Mutex::new(AppDataWhisperProcessor::default())); Ok(()) }) .invoke_handler(tauri::generate_handler![ - load_candle_model_whisper, + load_ort_model_whisper, ipc_audio_transcription, ]) .build() diff --git a/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/mod.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/mod.rs new file mode 100644 index 000000000..06066f55c --- /dev/null +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/mod.rs @@ -0,0 +1,27 @@ +pub mod whisper; + +use log::info; +use ort::execution_providers::{CUDAExecutionProvider, CoreMLExecutionProvider, ExecutionProvider}; +use tauri::Runtime; + +pub fn new_whisper_processor( + window: tauri::WebviewWindow, + model_type: Option, +) -> anyhow::Result { + let cuda = CUDAExecutionProvider::default().with_device_id(0); + let coreml = CoreMLExecutionProvider::default(); + + let whisper_model = model_type.unwrap_or_else(|| { + if cuda.is_available().unwrap_or(false) { + whisper::whisper::WhichModel::LargeV3 + } else if coreml.is_available().unwrap_or(false) { + whisper::whisper::WhichModel::Base + } else { + whisper::whisper::WhichModel::Tiny + } + }); + + info!("Loading whisper model: {:?}", whisper_model); + let (model_id, revision) = whisper_model.model_and_revision(); + whisper::whisper::WhisperPipeline::new(whisper_model, model_id, revision, window) +} diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/melfilters.bytes b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/melfilters.bytes similarity index 100% rename from crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/melfilters.bytes rename to crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/melfilters.bytes diff --git a/crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/melfilters128.bytes b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/melfilters128.bytes similarity index 100% rename from crates/tauri-plugin-ipc-audio-transcription-candle/src/models/whisper/melfilters128.bytes rename to crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/melfilters128.bytes diff --git a/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/mod.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/mod.rs new file mode 100644 index 000000000..1b004090d --- /dev/null +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/mod.rs @@ -0,0 +1,2 @@ +pub mod whisper; +pub mod whisper_processor; diff --git a/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper.rs new file mode 100644 index 000000000..61c7fea52 --- /dev/null +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper.rs @@ -0,0 +1,507 @@ +use std::{borrow::Cow, collections::HashMap, path::PathBuf}; + +use anyhow::{Result, anyhow}; +use clap::ValueEnum; +use hf_hub::{ + Repo, + RepoType, + api::sync::{Api, ApiBuilder}, +}; +use ndarray::{Array2, ArrayView3, Axis, s}; +use ort::{ + execution_providers::{ + CPUExecutionProvider, + CUDAExecutionProvider, + CoreMLExecutionProvider, + DirectMLExecutionProvider, + }, + session::{Session, SessionInputValue, builder::GraphOptimizationLevel}, + value::Value, +}; +use serde::Deserialize; +use tauri::Runtime; +use tokenizers::Tokenizer; + +use super::whisper_processor::WhisperProcessor; +use crate::helpers::huggingface::create_progress_emitter; + +// Helper function to provide a default value of true for serde +const fn default_true() -> bool { + true +} + +#[derive(Deserialize, Debug)] +pub struct WhisperConfig { + pub num_mel_bins: i64, + pub decoder_start_token_id: i64, + pub eos_token_id: i64, + // Corrected: Use serde default for missing is_multilingual field + #[serde(default = "default_true")] + pub is_multilingual: bool, + #[serde(default)] + pub no_timestamps_token_id: Option, + #[serde(default)] + pub lang_to_id: HashMap, +} + +#[derive(Debug)] +pub struct GenerationConfig { + pub language: Option, + pub task: String, + pub return_timestamps: bool, + pub max_new_tokens: usize, +} + +impl Default for GenerationConfig { + fn default() -> Self { + Self { + language: Some("en".to_string()), + task: "transcribe".to_string(), + return_timestamps: true, + max_new_tokens: 128, + } + } +} + +static WHISPER_LANGUAGES: std::sync::LazyLock> = + std::sync::LazyLock::new(|| { + let mut m = HashMap::new(); + m.insert("en", "english"); + m.insert("zh", "chinese"); + m.insert("de", "german"); + m.insert("es", "spanish"); + m.insert("ru", "russian"); + m.insert("ko", "korean"); + m.insert("fr", "french"); + m.insert("ja", "japanese"); + m.insert("pt", "portuguese"); + m.insert("tr", "turkish"); + m.insert("pl", "polish"); + m.insert("ca", "catalan"); + m.insert("nl", "dutch"); + m.insert("ar", "arabic"); + m.insert("sv", "swedish"); + m.insert("it", "italian"); + m.insert("id", "indonesian"); + m.insert("hi", "hindi"); + m.insert("fi", "finnish"); + m.insert("vi", "vietnamese"); + m.insert("he", "hebrew"); + m.insert("uk", "ukrainian"); + m.insert("el", "greek"); + m.insert("ms", "malay"); + m.insert("cs", "czech"); + m.insert("ro", "romanian"); + m.insert("da", "danish"); + m.insert("hu", "hungarian"); + m.insert("ta", "tamil"); + m.insert("no", "norwegian"); + m.insert("th", "thai"); + m.insert("ur", "urdu"); + m.insert("hr", "croatian"); + m.insert("bg", "bulgarian"); + m.insert("lt", "lithuanian"); + m.insert("la", "latin"); + m.insert("mi", "maori"); + m.insert("ml", "malayalam"); + m.insert("cy", "welsh"); + m.insert("sk", "slovak"); + m.insert("te", "telugu"); + m.insert("fa", "persian"); + m.insert("lv", "latvian"); + m.insert("bn", "bengali"); + m.insert("sr", "serbian"); + m.insert("az", "azerbaijani"); + m.insert("sl", "slovenian"); + m.insert("kn", "kannada"); + m.insert("et", "estonian"); + m.insert("mk", "macedonian"); + m.insert("br", "breton"); + m.insert("eu", "basque"); + m.insert("is", "icelandic"); + m.insert("hy", "armenian"); + m.insert("ne", "nepali"); + m.insert("mn", "mongolian"); + m.insert("bs", "bosnian"); + m.insert("kk", "kazakh"); + m.insert("sq", "albanian"); + m.insert("sw", "swahili"); + m.insert("gl", "galician"); + m.insert("mr", "marathi"); + m.insert("pa", "punjabi"); + m.insert("si", "sinhala"); + m.insert("km", "khmer"); + m.insert("sn", "shona"); + m.insert("yo", "yoruba"); + m.insert("so", "somali"); + m.insert("af", "afrikaans"); + m.insert("oc", "occitan"); + m.insert("ka", "georgian"); + m.insert("be", "belarusian"); + m.insert("tg", "tajik"); + m.insert("sd", "sindhi"); + m.insert("gu", "gujarati"); + m.insert("am", "amharic"); + m.insert("yi", "yiddish"); + m.insert("lo", "lao"); + m.insert("uz", "uzbek"); + m.insert("fo", "faroese"); + m.insert("ht", "haitian creole"); + m.insert("ps", "pashto"); + m.insert("tk", "turkmen"); + m.insert("nn", "nynorsk"); + m.insert("mt", "maltese"); + m.insert("sa", "sanskrit"); + m.insert("lb", "luxembourgish"); + m.insert("my", "myanmar"); + m.insert("bo", "tibetan"); + m.insert("tl", "tagalog"); + m.insert("mg", "malagasy"); + m.insert("as", "assamese"); + m.insert("tt", "tatar"); + m.insert("haw", "hawaiian"); + m.insert("ln", "lingala"); + m.insert("ha", "hausa"); + m.insert("ba", "bashkir"); + m.insert("jw", "javanese"); + m.insert("su", "sundanese"); + m + }); + +static WHISPER_TO_LANGUAGE_CODE: std::sync::LazyLock> = + std::sync::LazyLock::new(|| { + let mut m: HashMap<&'static str, &'static str> = WHISPER_LANGUAGES + .iter() + .map(|(k, v)| (*v, *k)) + .collect(); + m.insert("burmese", "my"); + m.insert("valencian", "ca"); + m.insert("flemish", "nl"); + m.insert("haitian", "ht"); + m.insert("letzeburgesch", "lb"); + m.insert("pushto", "ps"); + m.insert("panjabi", "pa"); + m.insert("moldavian", "ro"); + m.insert("moldovan", "ro"); + m.insert("sinhalese", "si"); + m.insert("castilian", "es"); + m + }); + +pub fn whisper_language_to_code(language: &str) -> Result { + let lower_lang = language.to_lowercase(); + if let Some(&code) = WHISPER_TO_LANGUAGE_CODE.get(lower_lang.as_str()) { + return Ok(code.to_string()); + } + if WHISPER_LANGUAGES.contains_key(lower_lang.as_str()) { + return Ok(lower_lang); + } + Err(anyhow!("Language '{}' is not supported.", language)) +} + +pub struct Whisper { + encoder_session: Session, + decoder_session: Session, + config: WhisperConfig, +} + +impl Whisper { + pub fn new( + model_id: &str, + revision: &str, + window: tauri::WebviewWindow, + ) -> Result { + let cache_api = hf_hub::Cache::from_env(); + let cache_repo = cache_api.repo(Repo::with_revision( + model_id.to_string(), + RepoType::Model, + revision.to_string(), + )); + + let api = ApiBuilder::new().with_progress(false).build()?; + let repo = api.repo(Repo::with_revision( + model_id.to_string(), + RepoType::Model, + revision.to_string(), + )); + + let encoder_model_path_sub_name = "onnx/encoder_model.onnx"; + let encoder_model_path = match cache_repo.get(encoder_model_path_sub_name) { + None => repo.download_with_progress( + encoder_model_path_sub_name, + create_progress_emitter( + window.clone(), + "tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress", + encoder_model_path_sub_name.to_string(), + ), + )?, + Some(p) => p, + }; + + let decoder_model_path_sub_name = "onnx/decoder_model.onnx"; + let decoder_model_path = match cache_repo.get(decoder_model_path_sub_name) { + Some(path) => path, + None => repo.download(decoder_model_path_sub_name)?, + }; + + let config_path_sub_name = "config.json"; + let config_path = match cache_repo.get(config_path_sub_name) { + Some(path) => path, + None => repo.download(config_path_sub_name)?, + }; + + let tokenizer_config_path_sub_name = "tokenizer.json"; + let tokenizer_config_path = match cache_repo.get(tokenizer_config_path_sub_name) { + Some(path) => path, + None => repo.download(tokenizer_config_path_sub_name)?, + }; + + let encoder_session = Self::create_optimized_session(encoder_model_path)?; + let decoder_session = Self::create_optimized_session(decoder_model_path)?; + + let mut config: WhisperConfig = serde_json::from_str(&std::fs::read_to_string(config_path)?)?; + let tokenizer_config: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(tokenizer_config_path)?)?; + + if let Some(added_tokens) = tokenizer_config["added_tokens"].as_array() { + let lang_to_id = added_tokens + .iter() + .filter_map(|token| { + let content = token["content"].as_str()?; + let id = token["id"].as_i64()?; + if content.starts_with("<|") + && content.ends_with("|>") + && content.len() >= 4 + && content.len() <= 6 + { + Some((content.to_string(), id)) + } else { + None + } + }) + .collect(); + config.lang_to_id = lang_to_id; + } + + Ok(Self { + encoder_session, + decoder_session, + config, + }) + } + + fn create_optimized_session(model_path: PathBuf) -> Result { + let session = Session::builder()? + .with_optimization_level(GraphOptimizationLevel::Level3)? + .with_parallel_execution(true)? + .with_execution_providers([ + CUDAExecutionProvider::default() + .with_device_id(0) + .build(), + CoreMLExecutionProvider::default().build(), + DirectMLExecutionProvider::default() + .with_device_id(0) + .build(), + CPUExecutionProvider::default().build(), + ])? + .commit_from_file(model_path)?; + Ok(session) + } + + fn retrieve_init_tokens( + &self, + gen_config: &GenerationConfig, + ) -> Result> { + let mut init_tokens = vec![self.config.decoder_start_token_id]; + let task_id = if gen_config.task == "translate" { + 50358 + } else { + 50359 + }; + + if self.config.is_multilingual { + let lang = gen_config.language.as_deref().unwrap_or("en"); + let lang_code = whisper_language_to_code(lang)?; + let lang_token = format!("<|{lang_code}|>"); + let lang_token_id = self + .config + .lang_to_id + .get(&lang_token) + .ok_or_else(|| anyhow!("Language token not found for: {}", lang_token))?; + init_tokens.push(*lang_token_id); + } + init_tokens.push(task_id); + + if !gen_config.return_timestamps + && let Some(no_timestamps_id) = self.config.no_timestamps_token_id + { + init_tokens.push(no_timestamps_id); + } + Ok(init_tokens) + } + + pub fn generate( + &mut self, + input_features: ArrayView3, + gen_config: &GenerationConfig, + ) -> Result> { + let (batch_size, num_mel_bins, sequence_length) = input_features.dim(); + let expected_mel_bins = usize::try_from(self.config.num_mel_bins)?; + + if batch_size != 1 || num_mel_bins != expected_mel_bins { + return Err(anyhow!( + "Incorrect input feature shape. Expected [1, {}, ...], but got [{}, {}, {}]", + expected_mel_bins, + batch_size, + num_mel_bins, + sequence_length + )); + } + + let mut decoder_input_ids = self.retrieve_init_tokens(gen_config)?; + + let owned_input = input_features.to_owned(); + let inputs = vec![("input_features", Value::from_array(owned_input)?)]; + let encoder_outputs = self.encoder_session.run(inputs)?; + let encoder_hidden_states = encoder_outputs.get("last_hidden_state").unwrap(); + + let mut generated_tokens = Vec::new(); + + // KV Cache + // let num_decoder_layers = self.config.decoder_layers as usize; + // let head_dim = self.config.d_model / self.config.decoder_attention_heads; + // let mut past_key_values: Vec> = (0..num_decoder_layers * 2) + // .map(|_| Array4::::zeros((1, self.config.decoder_attention_heads as usize, 0, head_dim as usize))) + // .collect(); + + for _step in 0..gen_config.max_new_tokens { + let decoder_input_ids_array = + Array2::from_shape_vec((1, decoder_input_ids.len()), decoder_input_ids.clone())? + .mapv(|x| x); + + // KV Cache + // let mut decoder_inputs: Vec<(Cow<'_, str>, SessionInputValue<'_>)> = Vec::with_capacity(2 + past_key_values.len()); + + let decoder_inputs: Vec<(Cow<'_, str>, SessionInputValue<'_>)> = vec![ + // name = encoder_hidden_states, type = tensor: float32[batch_size,encoder_sequence_length / 2,512] + ("encoder_hidden_states".into(), encoder_hidden_states.into()), + // name = input_ids, type = tensor: int64[batch_size,decoder_sequence_length] + ( + "input_ids".into(), + Value::from_array(decoder_input_ids_array)?.into(), + ), + ]; + + let decoder_outputs = self.decoder_session.run(decoder_inputs)?; + let logits_ref = decoder_outputs.get("logits").unwrap().view(); + let logits_view = logits_ref.try_extract_array::()?; + let next_token_logits = logits_view.slice(s![0, -1, ..]); + + let next_token = next_token_logits + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap()) + .map(|(index, _)| i64::try_from(index).unwrap()) + .unwrap(); + + if next_token == self.config.eos_token_id { + break; + } + + generated_tokens.push(next_token); + decoder_input_ids = vec![next_token]; + } + + Ok(generated_tokens) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum WhichModel { + Tiny, + Base, + Small, + Medium, + LargeV3, + LargeV3Turbo, +} + +impl WhichModel { + pub const fn model_and_revision(self) -> (&'static str, &'static str) { + match self { + Self::Tiny => ("onnx-community/whisper-tiny-ONNX", "main"), + Self::Base => ("onnx-community/whisper-base-ONNX", "main"), + Self::Small => ("onnx-community/whisper-small-ONNX", "main"), + Self::Medium => ("onnx-community/whisper-medium-ONNX", "main"), + Self::LargeV3 => ("onnx-community/whisper-large-v3-ONNX", "main"), + Self::LargeV3Turbo => ("onnx-community/whisper-large-v3-turbo-ONNX", "main"), + } + } +} + +/// A pipeline that encapsulates the full Whisper transcription process. +pub struct WhisperPipeline { + model: Whisper, + processor: WhisperProcessor, + tokenizer: Tokenizer, +} + +impl WhisperPipeline { + pub fn new( + which_model: WhichModel, + model_id: &str, + revision: &str, + window: tauri::WebviewWindow, + ) -> Result { + let model = Whisper::new(model_id, revision, window)?; + + // Initialize our new processor + let processor = WhisperProcessor::new(which_model)?; + + let api = Api::new()?; + let repo = api.repo(hf_hub::Repo::with_revision( + model_id.to_string(), + hf_hub::RepoType::Model, + revision.to_string(), + )); + let tokenizer_path = repo.get("tokenizer.json")?; + let tokenizer = Tokenizer::from_file(tokenizer_path) + .map_err(|e| anyhow!("Failed to load tokenizer: {}", e))?; + + Ok(Self { + model, + processor, + tokenizer, + }) + } + + pub fn transcribe( + &mut self, + audio: &[f32], + gen_config: &GenerationConfig, + ) -> Result { + // 1. Process the raw audio into a mel spectrogram with the correct shape [80, 3000] for normal, and [128, 3000] for large-v3 + let input_features = self.processor.process(audio); + + // 2. Add the batch dimension, making the shape [1, 80, 3000] for normal, and [1, 128, 3000] for large-v3 + let input_features = input_features.insert_axis(Axis(0)); + + // 3. Generate tokens. This will now work without a shape error. + let generated_tokens = self + .model + .generate(input_features.view(), gen_config)?; + + // The rest of the function remains the same... + let generated_tokens_u32: Vec = generated_tokens + .iter() + .map(|&x| u32::try_from(x).unwrap()) + .collect(); + + let transcript = self + .tokenizer + .decode(&generated_tokens_u32, true) + .map_err(|e| anyhow!("Failed to decode tokens: {}", e))?; + + Ok(transcript) + } +} diff --git a/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper_processor.rs b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper_processor.rs new file mode 100644 index 000000000..b420948cd --- /dev/null +++ b/crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper_processor.rs @@ -0,0 +1,134 @@ +use anyhow::{Result, anyhow}; +use byteorder::{ByteOrder, LittleEndian}; +use ndarray::{Array1, Array2, s}; +use rustfft::{FftPlanner, num_complex::Complex}; + +use super::whisper::WhichModel; + +// Constants from the Whisper paper/implementation +const SAMPLE_RATE: usize = 16000; +const N_FFT: usize = 400; +const N_MELS: usize = 80; // <--- Crucial: Use 80 for base, 128 for large-v3 +const HOP_LENGTH: usize = 160; +const CHUNK_LENGTH: usize = 30; +const N_SAMPLES: usize = CHUNK_LENGTH * SAMPLE_RATE; // 480000 samples +const N_FRAMES: usize = N_SAMPLES / HOP_LENGTH; // 3000 frames + +pub struct WhisperProcessor { + mel_filters: Array2, +} + +impl WhisperProcessor { + pub fn new(which_model: WhichModel) -> Result { + // Load the 80-bin mel filters + if which_model == WhichModel::LargeV3 || which_model == WhichModel::LargeV3Turbo { + let mel_bytes = include_bytes!("./melfilters128.bytes"); + let mut mel_filters_vec = vec![0f32; mel_bytes.len() / 4]; + LittleEndian::read_f32_into(mel_bytes, &mut mel_filters_vec); + + // The shape is [80, 201]. N_FFT/2 + 1 = 201. + let mel_filters = Array2::from_shape_vec((N_MELS, N_FFT / 2 + 1), mel_filters_vec) + .map_err(|e| anyhow!("Failed to create mel filters array: {}", e))?; + + Ok(Self { mel_filters }) + } else { + let mel_bytes = include_bytes!("./melfilters.bytes"); + let mut mel_filters_vec = vec![0f32; mel_bytes.len() / 4]; + LittleEndian::read_f32_into(mel_bytes, &mut mel_filters_vec); + + // The shape is [80, 201]. N_FFT/2 + 1 = 201. + let mel_filters = Array2::from_shape_vec((N_MELS, N_FFT / 2 + 1), mel_filters_vec) + .map_err(|e| anyhow!("Failed to create mel filters array: {}", e))?; + + Ok(Self { mel_filters }) + } + } + + /// Processes raw audio PCM data into a mel spectrogram. + pub fn process( + &self, + audio: &[f32], + ) -> Array2 { + // 1. Pad or truncate the audio to 30 seconds + let mut pcm_data = audio.to_vec(); + if pcm_data.len() < N_SAMPLES { + pcm_data.resize(N_SAMPLES, 0.0); + } else { + pcm_data.truncate(N_SAMPLES); + } + let pcm_data = Array1::from_vec(pcm_data); + + // 2. Compute the Short-Time Fourier Transform (STFT) + let stft = self.stft(&pcm_data); + // 3. Apply the mel filter bank + let mel_spectrogram = self.mel_filters.dot(&stft); + // 4. Apply logarithmic scaling + self.log_mel_spectrogram(&mel_spectrogram) + } + + /// Computes the Short-Time Fourier Transform (STFT) of the input audio. + fn stft( + &self, + pcm_data: &Array1, + ) -> Array2 { + // Create a Hann window + let window: Array1 = Array1::from_shape_fn(N_FFT, |i| { + 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / N_FFT as f32).cos()) + }); + + // Pad the input data + let mut padded_data = Array1::zeros(pcm_data.len() + N_FFT); + + let end = padded_data.len() - N_FFT / 2; + padded_data + .slice_mut(s![N_FFT / 2..end]) + .assign(pcm_data); + + let frames = padded_data + .windows(N_FFT) + .into_iter() + .step_by(HOP_LENGTH); + + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_forward(N_FFT); + + // Process each frame + let mut stft_result = Array2::zeros((N_FFT / 2 + 1, N_FRAMES)); + for (i, frame) in frames.enumerate() { + if i >= N_FRAMES { + break; + } + + // Apply window + let windowed_frame = frame.to_owned() * window.view(); + + // Prepare buffer for FFT + let mut buffer: Vec> = windowed_frame + .iter() + .map(|&x| Complex { re: x, im: 0.0 }) + .collect(); + + // Perform FFT + fft.process(&mut buffer); + + // Compute magnitude and store it + for j in 0..=(N_FFT / 2) { + stft_result[[j, i]] = buffer[j].norm_sqr().sqrt(); + } + } + + stft_result + } + + /// Converts a mel spectrogram to a log-scaled mel spectrogram. + fn log_mel_spectrogram( + &self, + mel_spec: &Array2, + ) -> Array2 { + let mut log_spec = mel_spec.mapv(|x| x.max(1e-10).log10()); + log_spec = + log_spec.mapv(|x| x.max(log_spec.fold(f32::NEG_INFINITY, |acc, &v| acc.max(v)) - 8.0)); + log_spec = (log_spec + 4.0) / 4.0; + log_spec + } +} diff --git a/crates/tauri-plugin-ipc-audio-vad-ort/Cargo.toml b/crates/tauri-plugin-ipc-audio-vad-ort/Cargo.toml index da856a8e1..21fc4f9ff 100644 --- a/crates/tauri-plugin-ipc-audio-vad-ort/Cargo.toml +++ b/crates/tauri-plugin-ipc-audio-vad-ort/Cargo.toml @@ -42,7 +42,7 @@ tauri-plugin = { version = "2.2", features = [ "build" ] } ort = { version = "2.0.0-rc.10", features = ["ndarray", "coreml"] } [target.'cfg(target_os = "windows")'.dependencies] -ort = { version = "2.0.0-rc.10", features = ["ndarray", "directml"] } +ort = { version = "2.0.0-rc.10", features = ["ndarray", "directml", "cuda"] } [target.'cfg(target_os = "linux")'.dependencies] -ort = { version = "2.0.0-rc.10", features = ["ndarray"] } +ort = { version = "2.0.0-rc.10", features = ["ndarray", "cuda"] }