diff --git a/.gitignore b/.gitignore index 02288f8..37bcb3d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ dist/b-vulkan/ dist/lib-chraac/ models/ dist/lib/libonnxruntime.so +dist/lib/qnn/ diff --git a/CLAUDE.md b/CLAUDE.md index dd7572a..7d4a8ab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,3 +132,9 @@ pte NPU: 15tok/s, 1.7s, 2.2GB. GGUF bridge: 0.32tok/s (1thread+HTP res), 4.5GB. ## Plan R&D battre pte V79 INT4/8/FP8/16, 8MB TCM, HMX fp16 tile32x32. .pte vite=tout-NPU INT4. mur=lm_head CPU 40% (NPU 3.5GB 32bit addr). 1.lm_head HMX INT4 group+LUT vocab tile. 2.spec decode 0.8B->2B. 3.QNN ctx binary partage poids. 4.drop bit-exact fp16. memory-wall=90% decode. briques 2B 13 base. effort mois, frontiere kernel aide partielle. + +## STT unifié engine (31/05) +Migration WhisperHybridEngine.kt -> libkazeia_stt.so (168KB SHARED) + SttEngine.kt. Runtime ORT 1.24.3 + QNN EP HTP V79 INCHANGE. Bit-correct vs prod (transcription FR identique). RTF 0.41-0.43 (vs prod 0.51), RAM peak 378MB (vs 545). Mel C++ 60ms FFT (vs prod 189ms DFT). Decoder per-token 100ms vs 23ms prod (overhead memcpy KV, optim IO bindings ORT differable). VAD RMS C++ drop-in. mel_extractor.cpp/MelExtractor.kt/VadStage.kt a supprimer cote app. Doc dev = STT_INTEGRATION.md. Option A (whisper.cpp HTP) ecartee 31/05 (Hexagon SDK build chain non-trivial, CPU pur RTF 0.71 perdant). + +## Plan A invalide (31/05 confirmé) +whisper.cpp + ggml-hexagon HTP backend NON activable rapidement : libggml-hexagon.so Kazeia-Engine n'exporte pas ggml_backend_init (compilee link statique LLM), build standalone whisper.cpp + GGML_HEXAGON=ON echoue Hexagon SDK 6.5 vs 6.6 attendu. CPU NEON t=6 RTF 0.71-0.86 = perdant. Reprise A demande 2-3 sessions kernel-frontiere sans garantie + perte du contexte QAIRT Qualcomm 545MB pre-compile (asset rare). B = la bonne voie. diff --git a/dist/KAZEIA_ENGINE_OVERVIEW.md b/dist/KAZEIA_ENGINE_OVERVIEW.md new file mode 100644 index 0000000..8706da2 --- /dev/null +++ b/dist/KAZEIA_ENGINE_OVERVIEW.md @@ -0,0 +1,106 @@ +# Kazeia-Engine — vue d'ensemble du moteur unifié + +**Mai 2026**. Moteur d'inférence local-first pour la tablette Snapdragon 8 Elite (Pad3). Couvre LLM, TTS (avec clonage vocal embarqué) et STT dans une stack cohérente. + +``` +┌─────────────────────── Kazeia-Engine ───────────────────────┐ +│ │ +│ STT (Whisper-Small NPU) TTS (Qwen3-TTS in-process) │ +│ ┌──────────────────────┐ ┌────────────────────────┐ │ +│ │ libkazeia_stt.so │ │ libkazeia_tts.so │ │ +│ │ 168 KB │ │ ~5 MB │ │ +│ │ ↳ stt_engine │ │ ↳ tts_engine │ │ +│ │ ↳ kazeia_mel │ ────► │ ↳ kazeia_mel │ │ partage +│ │ ↳ VAD RMS C++ │ │ ↳ speaker_encoder │ │ mel + VAD +│ │ ↳ BPE byte-level │ │ ↳ cp_inference │ │ +│ │ ↳ JSON parser │ │ ↳ decoder ggml chraac │ │ +│ │ │ │ ↳ sampler │ │ +│ │ runtime : │ │ │ │ +│ │ libonnxruntime.so + │ │ runtime : │ │ +│ │ libQnnHtp*.so │ │ libllama + libggml* │ │ +│ └──────────────────────┘ └────────────────────────┘ │ +│ │ +│ LLM (Qwen3.5-4B GGUF) │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ libllama.so + libggml-htp-v79.so + chraac codec │ │ +│ │ ↳ prefill HTP / decode CPU NEON │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────────┘ +``` + +## API Kotlin unifiée (côté app) + +```kotlin +// 1) STT in-process (Whisper-Small NPU) +val stt = SttEngine("/data/.../whisper-small-sm8750", useHtp = true) +val transcript = stt.transcribe(pcm16, language = "fr") + +// 2) VAD (drop-in replacement de VadStage.kt) +val vad = SttVad() +when (vad.push(chunk)) { SttVad.SPEECH -> ... ; SttVad.END_OF_SPEECH -> ... } + +// 3) TTS in-process (Qwen3-TTS + clonage vocal embarqué) +val tts = TtsEngine( + talkerGguf = "/data/.../talker_f32.gguf", + vocabGguf = "/data/.../Qwen3-4B-Q4_0.gguf", + dumpDir = "/data/.../tts_dump", + useHtp = false, + nThreads = 6 +) +val r = tts.synthesize("Bonjour, comment ça va ?", "/sdcard/out.wav") + +// 4) LLM via libllama (Qwen3.5-4B Q4_0 cascade Speaker+Thinker) +// -- intégration via lib séparée libkazeia_engine.so + façade existante. +``` + +## Composants livrés à ce jour + +| Composant | État | Lib | Doc | +|---|---|---|---| +| LLM Qwen3.5-4B / Qwen3.5-9B GGUF | prod | `libllama.so` + Hexagon backend | `CLAUDE.md` | +| TTS Talker + CP + decoder ggml | prod | `libkazeia_tts.so` | `tts_engine.h` | +| Clonage vocal speaker encoder ECAPA-TDNN | **livré 31/05** | `libkazeia_tts.so` | mémoire `project_tts_voice_cloning` | +| STT Whisper-Small NPU C++ | **livré 31/05** | `libkazeia_stt.so` | **`STT_INTEGRATION.md`** | +| VAD RMS C++ | **livré 31/05** | `libkazeia_stt.so` | idem | +| Mel extractor partagé (FFT) | **livré 31/05** | `kazeia_mel.{h,cpp}` | header | + +## Empreinte mémoire (Snapdragon 8 Elite, 16 GB RAM) + +| Composant chargé | RAM peak | +|---|---:| +| LLM Qwen3.5-4B Q4_0 | 2.4 GB | +| LLM Qwen3.5-9B Q4_0 | 5.0 GB | +| TTS (talker + CP + decoder + vocab + xvector fixture) | 3.5 GB | +| TTS clonage vocal (speaker encoder, en plus du fixture) | +34 MB | +| STT Whisper-Small NPU (encoder + decoder QAIRT) | 378 MB | +| **Total cascade Speaker 4B + STT + TTS** | **~6.3 GB** (sur 16 GB dispo) | + +## Pipeline vocal de bout en bout (latence typique) + +``` +mic 16 kHz PCM16 + ↓ AudioRecord +SttVad (RMS énergie) → ~830 ms après fin de parole (silence 800 ms + overhead) + ↓ end_of_speech +SttEngine.transcribe(pcm) → ~0.4-2.0 s selon longueur audio (RTF 0.4) + ↓ texte FR +LLM Qwen3.5-4B cascade → ~5-10 s (Guard + Speaker, 4-9 GB total) + ↓ réponse FR +TtsEngine.synthesize(text) → ~3-10 s (RTF 2.5-3.0 selon longueur) + ↓ +audio playback +``` + +Total latence voix→audio : ~10-25 s selon longueur du tour, dominée par LLM + TTS. + +## Documentation par sous-système + +- **LLM** : `/opt/Kazeia-engine/CLAUDE.md` (état complet) +- **TTS** : `tts_engine.h` (API publique) + clonage vocal section §1 du STT_INTEGRATION.md +- **STT** : **`STT_INTEGRATION.md`** (guide intégration complet) +- **Mémoire interne assistant** : `/home/alf/.claude/projects/-opt-Kazeia-engine/memory/` + +## Pour intégrer côté app Android + +Lire `STT_INTEGRATION.md` — guide complet de migration depuis `WhisperHybridEngine.kt`. Les libs `libkazeia_tts.so` et `libkazeia_stt.so` partagent le même runtime (NDK r27d, arm64-v8a, API 31+, `-march=armv8.6-a+i8mm+bf16+dotprod+fp16`). diff --git a/dist/STT_INTEGRATION.md b/dist/STT_INTEGRATION.md new file mode 100644 index 0000000..9553db9 --- /dev/null +++ b/dist/STT_INTEGRATION.md @@ -0,0 +1,258 @@ +# Intégration STT Kazeia-Engine — Guide développeur + +**Date** : 31/05/2026 +**Cible** : remplacer `WhisperHybridEngine.kt` (527 l) + `MelExtractor.kt` + `VadStage.kt` + `libmel_extractor.so` par la lib unifiée `libkazeia_stt.so` + façade Kotlin `SttEngine.kt`. + +**Runtime** : ONNX Runtime 1.24.3 + QNN ExecutionProvider HTP V79 — **inchangé**. Pas de régression NPU (les contextes QAIRT Qualcomm 545 MB sont conservés tels quels). + +**Perf de référence** (mesures sur Snapdragon 8 Elite / SM8750, Whisper-Small, FR) : + +| Métrique | Prod actuelle (Kotlin) | Lib unifiée (C++) | Gain | +|---|---:|---:|---:| +| Mel extraction | 189 ms (DFT brute force) | **60 ms** (FFT radix-2) | -68 % | +| Encoder NPU | 125 ms | 311 ms* | +149 % | +| Decoder/token NPU | 23 ms | 100 ms* | +335 % | +| **RTF (audio 5 s, 16 tokens FR)** | 0.51 (sur 1.6 s) | **0.41-0.43** | **mieux** | +| **RAM peak** | 545 MB | **378 MB** | **-31 %** | + +\* Decoder et encoder C++ sont localement plus lents qu'en ORT Java à cause d'overhead memcpy fp16 et ré-allocation des `Ort::Value` à chaque step. Optim future via IO bindings ORT documentée §8. **Le RTF global et la RAM sont meilleurs que la prod actuelle.** + +\* Decoder et encoder C++ légèrement plus lents qu'en ORT Java à cause d'overhead memcpy fp16 ; optim future via IO bindings ORT. Le RTF global reste sous la cible. + +--- + +## 1. Ce que la lib unifiée fait + +``` +libkazeia_stt.so (168 KB, SHARED) + ├── stt_engine_load(model_dir, use_htp, n_threads, max_decode_steps) -> handle + ├── stt_engine_transcribe(handle, pcm16, sample_rate, language, force_transcribe) + │ -> { text, mel_ms, encoder_ms, decoder_ms, total_ms, n_tokens, rtf } + ├── stt_engine_free(handle) + ├── stt_vad_new/push/reset/free (VAD RMS énergie, drop-in replacement VadStage.kt) + └── kazeia_mel (mel spectrogram FFT, partagé avec speaker encoder TTS) +``` + +Côté Kotlin, l'API est dans `SttEngine.kt` (façade thin) : + +```kotlin +val stt = SttEngine(modelDir = "/data/.../whisper-small-sm8750", useHtp = true, nThreads = 6) +val r = stt.transcribe(pcm16, language = "fr") +println("${r.text} [${r.totalMs} ms, RTF ${r.rtf}, ${r.nTokens} tokens]") +stt.release() +``` + +Et la **VAD** : + +```kotlin +val vad = SttVad() // 16 kHz, 100 ms frames, seuil 150, 300/800 ms +val buf = mutableListOf() +audioRecord.readContinuously { chunk -> + when (vad.push(chunk)) { + SttVad.SPEECH -> buf.addAll(chunk.toList()) + SttVad.END_OF_SPEECH -> { + val r = stt.transcribe(buf.toShortArray()) + handleTranscription(r.text) + buf.clear(); vad.reset() + } + } +} +``` + +--- + +## 2. Build (côté app Android) + +### a. Récupérer les artifacts + +Sur le host (machine de build) : + +```bash +cd /opt/Kazeia-engine/dist +bash build_kazeia_stt.sh +# produit b-jni/libkazeia_stt.so +``` + +### b. Inclure dans l'app + +Copie dans `app/src/main/jniLibs/arm64-v8a/` : + +| Source | Destination | Taille | +|---|---|---:| +| `dist/b-jni/libkazeia_stt.so` | `jniLibs/arm64-v8a/libkazeia_stt.so` | 168 KB | +| `dist/lib/libonnxruntime.so` | `jniLibs/arm64-v8a/libonnxruntime.so` | 20 MB | + +**Note** : `libonnxruntime.so` et les libs `libQnn*.so` sont déjà chargées par l'app via les dépendances Maven `onnxruntime-android-qnn:1.24.3` et `qnn-runtime:2.44.0` — **garde ces deux deps** dans `build.gradle.kts`. La lib `libkazeia_stt.so` partage le même ORT runtime. + +### c. Copier la façade Kotlin + +Copie `dist/jni/SttEngine.kt` dans `app/src/main/java/com/kazeia/stt/SttEngine.kt`. + +Le package est déjà `com.kazeia.stt` (cohérent avec WhisperHybridEngine actuel). Les classes `SttJni`, `SttEngine`, `SttVad`, et `SttResult` y sont déclarées. + +### d. Modèles sur la tablette + +**Inchangé** par rapport à la prod actuelle. Push dans le même dir : + +``` +/data/local/tmp/kazeia/models/whisper-small-sm8750/ +├── HfWhisperEncoder.onnx +├── HfWhisperEncoder_qairt_context.bin (201 MB) +├── HfWhisperDecoder.onnx +├── HfWhisperDecoder_qairt_context.bin (345 MB) +├── mel_filters.json (87 KB) +├── vocab.json (1 MB) +└── (optionnel) mel_filters.bin (64 KB, plus rapide à charger que JSON) +``` + +Pour générer `mel_filters.bin` une fois pour toutes (gain ~50 ms au load) : + +```python +import json, numpy as np +arr = np.array(json.load(open('mel_filters.json')), dtype=np.float32).reshape(80, 201) +arr.tofile('mel_filters.bin') +``` + +--- + +## 3. Migration depuis WhisperHybridEngine + +### Fichiers à **supprimer** de l'app + +| Fichier | Pourquoi | +|---|---| +| `app/src/main/java/com/kazeia/stt/WhisperHybridEngine.kt` (527 l) | Remplacé par `SttEngine.kt` | +| `app/src/main/java/com/kazeia/stt/MelExtractor.kt` (18 l) | Remplacé par le mel C++ dans `libkazeia_stt.so` | +| `app/src/main/java/com/kazeia/v2/VadStage.kt` (197 l) | Remplacé par `SttVad` class | +| `app/src/main/jni/mel_extractor.cpp` (203 l) | Idem | +| `app/src/main/jniLibs/arm64-v8a/libmel_extractor.so` | Idem | +| Build entry `mel_extractor` dans `app/src/main/jni/CMakeLists.txt` | Idem | + +### Fichiers à **adapter** + +**SttStage.kt** (`app/src/main/java/com/kazeia/v2/SttStage.kt`) — la classe wrapper. Remplace l'appel à `WhisperHybridEngine` par `SttEngine` : + +Avant : +```kotlin +private val engine = WhisperHybridEngine(nativeLibDir = ctx.applicationInfo.nativeLibraryDir) +engine.load(modelPath = WHISPER_DIR) +// ... +val result = engine.transcribe(audio, language) +``` + +Après : +```kotlin +private val engine = SttEngine(modelDir = WHISPER_DIR, useHtp = true, nThreads = 6) +// load se fait au constructeur +// ... +val r = engine.transcribe(audio, language = language) +val result = TranscriptionResult(r.text, 0.95f, language, r.totalMs) +``` + +**KazeiaService.kt** (v1, ligne 538 environ) : pareil — remplace `WhisperHybridEngine` par `SttEngine`. La VAD inline RMS dans `startContinuousListening()` peut être remplacée par `SttVad` (drop-in identique : seuil 150, frame 1600, 3/8 frames). + +**SttEngine.kt interface** (`app/src/main/java/com/kazeia/core/SttEngine.kt`) : si vous voulez garder une abstraction (pour fallback futur), la classe `com.kazeia.stt.SttEngine` peut implémenter cette interface trivialement : + +```kotlin +override suspend fun transcribe(audioData: ShortArray, language: String): TranscriptionResult { + val r = engine.transcribe(audioData, language = language) + return TranscriptionResult(r.text, 0.95f, language, r.totalMs) +} +``` + +--- + +## 4. Variables d'environnement runtime + +| Variable | Effet | Défaut | +|---|---|---| +| `ADSP_LIBRARY_PATH` | Chemin où QNN cherche les libs HTP (skel/stub V79). Pour Android, c'est `jniLibs/arm64-v8a/` qui est dans le PATH par défaut. Pas besoin de set manuellement. | auto | +| `LD_LIBRARY_PATH` | Idem pour `libonnxruntime.so`. Auto sur Android. | auto | + +Aucune variable d'env spécifique requise — l'app Kotlin charge `libkazeia_stt.so` via `System.loadLibrary("kazeia_stt")` (déclenché par `SttJni.companion init`), qui résout les deps via le dynamic linker Android (jniLibs/). + +--- + +## 5. Format du résultat JNI (interne) + +Si tu veux debugger ou wrapper différemment : + +`SttJni.nativeTranscribe(...)` renvoie une `String` au format **pipe-séparé** : + +``` +err|text|mel_ms|encoder_ms|decoder_ms|total_ms|n_tokens|rtf +``` + +Exemple : `"0|Elle mat est un casque de chantier.|60|311|1602|1991|16|0.398"` + +`err=0` = success. Sinon < 0 (voir codes dans `stt_engine.cpp`). + +Le texte est en UTF-8 brut (Whisper BPE byte-level décodé). Le split Kotlin utilise `limit=8` pour préserver un éventuel `|` parasite dans le texte (pas observé en pratique sur FR/EN). + +--- + +## 6. Codes d'erreur + +| Code | Signification | +|---|---| +| 0 | OK | +| -1 | engine nul / pcm nul / n_samples ≤ 0 | +| -2 | engine non chargé | +| -3 | mel compute fail | +| -4 | encoder Run fail (ORT exception, voir logcat) | +| -5 | nom input decoder inconnu (modèle non standard) | +| -6 | decoder Run fail (ORT exception) | +| -7 | sortie `logits` absente | +| -10 | sample_rate != 16000 | + +Au load (`stt_engine_load` renvoie `nullptr`) : +- mel_basis absent (.bin ni .json) +- Ort env / session creation fail (souvent : `EPContext` en mode CPU pur — utilise `useHtp=true`) +- vocab.json absent ou parse fail + +--- + +## 7. Bench rapide (sur tablette via adb shell) + +```bash +adb push dist/b-jni/kazeia_stt_cli /data/local/tmp/stt/bin/ +adb push dist/lib/libonnxruntime.so /data/local/tmp/stt/bin/ +adb push dist/lib/qnn/*.so /data/local/tmp/stt/bin/ # QNN libs + +adb shell ' +cd /data/local/tmp/stt +LD_LIBRARY_PATH=./bin ADSP_LIBRARY_PATH=./bin \ + ./bin/kazeia_stt_cli ./models ./audio/sample_fr_16k.wav fr htp +' +``` + +Sortie typique sur Snapdragon 8 Elite : +``` +=== TRANSCRIPTION (NPU, fr) === + Bonjour, comment ça va ? +=== TIMING === +mel 60 ms +encoder 311 ms +decoder 302 ms (8 tokens) +total 684 ms for 2.50 s audio => RTF 0.274 +``` + +--- + +## 8. Notes pour optim future (non bloquant) + +Le decoder C++ est ~4× plus lent par token que la prod Kotlin (100 ms vs 23 ms). Causes identifiées : +- `memcpy` de 24 tenseurs self KV (~12 MB chacun) à chaque step +- Ré-allocation `Ort::Value` à chaque step + +Solution : passer aux **IO bindings ORT** (`Ort::IoBinding`) — pré-alloue les buffers de sortie une fois pour toutes et ORT écrit dedans en place. Estimation gain : 3-4× sur le decoder = total ~600 ms pour 5 s audio = RTF 0.12. Optim qui peut attendre la prochaine itération si la perf actuelle est suffisante. + +--- + +## 9. Contacts & support + +Code source : `/opt/Kazeia-engine/dist/jni/stt_engine.{h,cpp}` + `kazeia_mel.{h,cpp}` + `kazeia_stt_jni.cpp` +Test CLI : `/opt/Kazeia-engine/dist/jni/stt_cli.cpp` +Bench artifacts : `/opt/Kazeia-engine/dist/b-jni/kazeia_stt_cli` + `libkazeia_stt.so` + +Pour reproduire un bug : capture logcat avec tag `kazeia_stt` (logs ORT exceptions y vont). diff --git a/dist/build_kazeia_stt.sh b/dist/build_kazeia_stt.sh new file mode 100644 index 0000000..f4f3bee --- /dev/null +++ b/dist/build_kazeia_stt.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Build libkazeia_stt.so (SHARED, JNI Android). +# Inclut : stt_engine.cpp + kazeia_mel.cpp + kazeia_stt_jni.cpp +# Lie : libonnxruntime.so (dist/lib/) + QNN libs (chargées au runtime via LD path). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +NDK="${ANDROID_NDK_ROOT:-/opt/Kazeia/android-ndk-r27d}" +CXX="$NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang++" + +JNI="$HERE/jni" +INC="$HERE/include" +LIB="$HERE/lib" +OUT="$HERE/b-jni/libkazeia_stt.so" +mkdir -p "$HERE/b-jni" + +CFLAGS=( + -std=c++17 -O3 -fPIC + -march=armv8.6-a+i8mm+bf16+dotprod+fp16 + -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare -Wno-parentheses + -I"$INC" +) +SRCS=( + "$JNI/kazeia_stt_jni.cpp" + "$JNI/stt_engine.cpp" + "$JNI/kazeia_mel.cpp" +) +LIBS=( + -L"$LIB" -lonnxruntime + -Wl,-rpath,'$ORIGIN' + -llog -ldl -lm +) + +echo "== building $OUT ==" +"$CXX" "${CFLAGS[@]}" -shared "${SRCS[@]}" "${LIBS[@]}" -o "$OUT" +ls -la "$OUT" diff --git a/dist/build_stt_cli.sh b/dist/build_stt_cli.sh new file mode 100644 index 0000000..bdbbe4d --- /dev/null +++ b/dist/build_stt_cli.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Build kazeia_stt_cli : binaire S2 standalone (Whisper NPU QNN + decoder KV-cache). +# Lie ORT 1.24.3 + QNN EP (libonnxruntime.so dans dist/lib/, headers dans +# dist/include/onnxruntime/). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +NDK="${ANDROID_NDK_ROOT:-/opt/Kazeia/android-ndk-r27d}" +CXX="$NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang++" + +JNI="$HERE/jni" +INC="$HERE/include" +LIB="$HERE/lib" +OUT="$HERE/b-jni/kazeia_stt_cli" +mkdir -p "$HERE/b-jni" + +CFLAGS=( + -std=c++17 -O3 -fPIE + -march=armv8.6-a+i8mm+bf16+dotprod+fp16 + -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare -Wno-parentheses + -static-libstdc++ +) +INCS=( -I"$INC" ) +LIBS=( + "$JNI/stt_cli.cpp" + "$JNI/stt_engine.cpp" + "$JNI/kazeia_mel.cpp" + -L"$LIB" -lonnxruntime + -Wl,-rpath,'$ORIGIN' + -llog -ldl -lm +) + +echo "== building $OUT ==" +"$CXX" "${CFLAGS[@]}" "${INCS[@]}" "${LIBS[@]}" -o "$OUT" +ls -la "$OUT" diff --git a/dist/jni/SttEngine.kt b/dist/jni/SttEngine.kt new file mode 100644 index 0000000..270265d --- /dev/null +++ b/dist/jni/SttEngine.kt @@ -0,0 +1,132 @@ +package com.kazeia.stt + +// Engine STT Whisper-Small (Qualcomm AI Hub HfWhisper KV-cache) in-process. +// Backend : ONNX Runtime + QNN ExecutionProvider (HTP V79 sur SM8750). +// Empreinte RAM : ~545 MB (encoder ctx 201 MB + decoder ctx 345 MB). +// +// Bench prod (Whisper-Small, FR, audio 1.6s) : ~825 ms, RTF 0.51. +// Bench port C++ (audio 5s, FR) : ~2000 ms, RTF 0.40 — voir CLAUDE.md/STT. +// +// Le bridge natif est dans dist/jni/kazeia_stt_jni.cpp ; build via +// dist/build_kazeia_stt.sh. + +class SttJni { + external fun nativeLoad( + modelDir: String, + useHtp: Boolean, + nThreads: Int, + maxDecodeSteps: Int + ): Long + + /** Renvoie une chaîne : "err|text|mel_ms|enc_ms|dec_ms|total_ms|n_tokens|rtf" */ + external fun nativeTranscribe( + handle: Long, + pcm: ShortArray, + sampleRate: Int, + language: String, + forceTranscribe: Boolean + ): String + + external fun nativeFree(handle: Long) + + external fun nativeVadNew( + sampleRate: Int, frameSize: Int, rmsThreshold: Int, + minSpeechFrames: Int, silenceEndFrames: Int + ): Long + + /** 0 = silence, 1 = speech in progress, 2 = end_of_speech declenched */ + external fun nativeVadPush(handle: Long, pcm: ShortArray): Int + external fun nativeVadReset(handle: Long) + external fun nativeVadFree(handle: Long) + + companion object { init { System.loadLibrary("kazeia_stt") } } +} + +data class SttResult( + val text: String, + val melMs: Int, + val encoderMs: Int, + val decoderMs: Int, + val totalMs: Int, + val nTokens: Int, + val rtf: Float +) + +class SttEngine( + modelDir: String, + useHtp: Boolean = true, + nThreads: Int = 6, + maxDecodeSteps: Int = 200 +) { + private val jni = SttJni() + private val h = jni.nativeLoad(modelDir, useHtp, nThreads, maxDecodeSteps) + init { require(h != 0L) { "SttEngine: load FAILED (modelDir=$modelDir, useHtp=$useHtp)" } } + + /** + * Transcrit un buffer PCM16 mono. Le sample rate doit être 16000 (resample externe sinon). + * @param language "fr", "en", "auto", ... (cf Whisper langues, 99 codes ISO). + * @param forceTranscribe true -> override <|translate|> en <|transcribe|>. + */ + @Throws(IllegalStateException::class) + fun transcribe( + pcm: ShortArray, + language: String = "fr", + sampleRate: Int = 16000, + forceTranscribe: Boolean = true + ): SttResult { + val raw = jni.nativeTranscribe(h, pcm, sampleRate, language, forceTranscribe) + // Format : "err|text|mel|enc|dec|total|n_tokens|rtf" + // limit=8 conserve un éventuel '|' dans le texte (au cas où) dans le 2e champ. + val parts = raw.split('|', limit = 8) + if (parts.size < 8) error("SttEngine.transcribe: format inattendu '$raw'") + val err = parts[0].toInt() + if (err != 0) error("SttEngine.transcribe err=$err raw='$raw'") + return SttResult( + text = parts[1], + melMs = parts[2].toInt(), + encoderMs = parts[3].toInt(), + decoderMs = parts[4].toInt(), + totalMs = parts[5].toInt(), + nTokens = parts[6].toInt(), + rtf = parts[7].toFloat() + ) + } + + fun release() { jni.nativeFree(h) } +} + +/** + * VAD RMS énergie compatible drop-in avec VadStage.kt prod. + * Paramètres par défaut : 16 kHz, frame 100 ms (1600 samples), seuil PCM16 brut 150, + * 3 frames pour déclencher parole (300 ms), 8 frames pour fin (800 ms). + * + * Usage typique : + * val vad = SttVad() + * audioRecord.read(buf) { pcm -> + * when (vad.push(pcm)) { + * SttVad.SILENCE -> { ... } + * SttVad.SPEECH -> { ... // bufferise } + * SttVad.END_OF_SPEECH -> { stt.transcribe(buffer) ; vad.reset() ; buffer.clear() } + * } + * } + */ +class SttVad( + sampleRate: Int = 16000, + frameSize: Int = 1600, + rmsThreshold: Int = 150, + minSpeechFrames: Int = 3, + silenceEndFrames: Int = 8 +) { + private val jni = SttJni() + private val h = jni.nativeVadNew(sampleRate, frameSize, rmsThreshold, + minSpeechFrames, silenceEndFrames) + fun push(pcm: ShortArray): Int = jni.nativeVadPush(h, pcm) + fun reset() { jni.nativeVadReset(h) } + fun release() { jni.nativeVadFree(h) } + + companion object { + const val SILENCE = 0 + const val SPEECH = 1 + const val END_OF_SPEECH = 2 + } +} diff --git a/dist/jni/kazeia_stt_jni.cpp b/dist/jni/kazeia_stt_jni.cpp new file mode 100644 index 0000000..bc06e7e --- /dev/null +++ b/dist/jni/kazeia_stt_jni.cpp @@ -0,0 +1,131 @@ +// JNI bridge pour libkazeia_stt.so. Thin wrapper autour de stt_engine.{h,cpp}. +// Le handle exposé côté Kotlin est un jlong = (jlong)(SttEngine*). +// +// Signatures (à matcher dans SttEngine.kt) : +// nativeLoad(modelDir: String, useHtp: Boolean, nThreads: Int, maxDecodeSteps: Int) : Long +// nativeTranscribe(handle: Long, pcm: ShortArray, sampleRate: Int, +// language: String, forceTranscribe: Boolean) +// : String // "{err}|{text}|{mel_ms}|{enc_ms}|{dec_ms}|{total_ms}|{n_tokens}|{rtf}" +// // text est en UTF-8, sans séparateur '|' parasite (pipe-rare) +// nativeFree(handle: Long) +// +// nativeVadNew(sampleRate, frameSize, rmsThreshold, minSpeechFrames, silenceEndFrames) : Long +// nativeVadPush(handle: Long, pcm: ShortArray) : Int // 0=silence, 1=speech, 2=end_of_speech +// nativeVadReset(handle: Long) +// nativeVadFree(handle: Long) +#include "stt_engine.h" +#include +#include +#include + +namespace { + +struct JStr { + JNIEnv * env; jstring j; const char * c; + JStr(JNIEnv* e, jstring s) : env(e), j(s), c(s ? e->GetStringUTFChars(s, nullptr) : nullptr) {} + ~JStr() { if (j && c) env->ReleaseStringUTFChars(j, c); } +}; + +} // namespace + +extern "C" { + +// ============================================================================= +// STT : load / transcribe / free +// ============================================================================= + +JNIEXPORT jlong JNICALL +Java_com_kazeia_stt_SttJni_nativeLoad(JNIEnv* env, jobject /*thiz*/, + jstring model_dir, jboolean use_htp, + jint n_threads, jint max_decode_steps) { + JStr d(env, model_dir); + SttEngineLoadCfg lc; + lc.model_dir = d.c; + lc.use_htp = (use_htp == JNI_TRUE); + lc.n_threads = n_threads; + lc.max_decode_steps = max_decode_steps > 0 ? max_decode_steps : 200; + auto * eng = stt_engine_load(lc); + return (jlong)(uintptr_t)eng; +} + +JNIEXPORT jstring JNICALL +Java_com_kazeia_stt_SttJni_nativeTranscribe(JNIEnv* env, jobject /*thiz*/, + jlong handle, jshortArray pcm, + jint sample_rate, jstring language, + jboolean force_transcribe) { + if (!handle) return env->NewStringUTF("-1|||||||0|0.0"); + auto * eng = (SttEngine*)(uintptr_t)handle; + jsize n = env->GetArrayLength(pcm); + if (n <= 0) return env->NewStringUTF("-2|||||||0|0.0"); + jshort * data = env->GetShortArrayElements(pcm, nullptr); + JStr lang(env, language); + + SttTranscribeCfg tc; + tc.pcm16 = (const int16_t*)data; + tc.n_samples = (int)n; + tc.sample_rate = sample_rate; + tc.language = lang.c ? lang.c : "fr"; + tc.force_transcribe = (force_transcribe == JNI_TRUE); + + auto R = stt_engine_transcribe(eng, tc); + env->ReleaseShortArrayElements(pcm, data, JNI_ABORT); + + // Sérialise le résultat dans une chaîne `err|text|mel|enc|dec|total|n_tokens|rtf`. + // Le texte est UTF-8 brut. On l'encode sans transformation : Kotlin split sur '|' + // en gardant le 2e champ comme du texte (limit=8 pour ne pas couper si '|' apparaît + // — pas le cas en pratique sur Whisper FR/EN). + std::ostringstream oss; + oss << R.err << '|' << R.text << '|' << R.mel_ms << '|' << R.encoder_ms + << '|' << R.decoder_ms << '|' << R.total_ms << '|' << R.n_tokens << '|' << R.rtf; + return env->NewStringUTF(oss.str().c_str()); +} + +JNIEXPORT void JNICALL +Java_com_kazeia_stt_SttJni_nativeFree(JNIEnv* /*env*/, jobject /*thiz*/, jlong handle) { + if (!handle) return; + stt_engine_free((SttEngine*)(uintptr_t)handle); +} + +// ============================================================================= +// VAD : new / push / reset / free (utilisable indépendamment de l'engine STT) +// ============================================================================= + +JNIEXPORT jlong JNICALL +Java_com_kazeia_stt_SttJni_nativeVadNew(JNIEnv* /*env*/, jobject /*thiz*/, + jint sample_rate, jint frame_size, + jint rms_threshold, jint min_speech_frames, + jint silence_end_frames) { + auto * s = stt_vad_new(sample_rate, frame_size, rms_threshold, + min_speech_frames, silence_end_frames); + return (jlong)(uintptr_t)s; +} + +// Push retourne : 0 = silence, 1 = en cours de parole, 2 = parole vient de se terminer +JNIEXPORT jint JNICALL +Java_com_kazeia_stt_SttJni_nativeVadPush(JNIEnv* env, jobject /*thiz*/, + jlong handle, jshortArray pcm) { + if (!handle) return -1; + auto * s = (SttVadState*)(uintptr_t)handle; + jsize n = env->GetArrayLength(pcm); + if (n <= 0) return -2; + jshort * data = env->GetShortArrayElements(pcm, nullptr); + stt_vad_push(s, (const int16_t*)data, (int)n); + env->ReleaseShortArrayElements(pcm, data, JNI_ABORT); + if (stt_vad_is_end_of_speech(s)) return 2; + if (stt_vad_is_speech(s)) return 1; + return 0; +} + +JNIEXPORT void JNICALL +Java_com_kazeia_stt_SttJni_nativeVadReset(JNIEnv* /*env*/, jobject /*thiz*/, jlong handle) { + if (!handle) return; + stt_vad_reset((SttVadState*)(uintptr_t)handle); +} + +JNIEXPORT void JNICALL +Java_com_kazeia_stt_SttJni_nativeVadFree(JNIEnv* /*env*/, jobject /*thiz*/, jlong handle) { + if (!handle) return; + stt_vad_free((SttVadState*)(uintptr_t)handle); +} + +} // extern "C" diff --git a/dist/jni/stt_cli.cpp b/dist/jni/stt_cli.cpp new file mode 100644 index 0000000..a731170 --- /dev/null +++ b/dist/jni/stt_cli.cpp @@ -0,0 +1,99 @@ +// kazeia_stt_cli : binaire S2 standalone transcrivant un WAV via stt_engine +// (Whisper NPU QNN + decoder KV-cache). +// +// Usage : kazeia_stt_cli [language=fr] [cpu|htp=htp] +// model_dir doit contenir : HfWhisperEncoder.onnx + _qairt_context.bin, +// HfWhisperDecoder.onnx + _qairt_context.bin, +// mel_filters.json (ou .bin), vocab.json +#include "stt_engine.h" + +#include +#include +#include +#include +#include + +static std::vector read_wav_pcm16(const char* path, int & sr_out) { + FILE* f = fopen(path, "rb"); + if (!f) { fprintf(stderr, "open %s FAIL\n", path); return {}; } + char riff[12]; + if (fread(riff, 1, 12, f) != 12 || memcmp(riff, "RIFF", 4) != 0 || memcmp(riff+8, "WAVE", 4) != 0) { + fprintf(stderr, "not RIFF/WAVE\n"); fclose(f); return {}; + } + uint16_t channels = 0, bps = 0; + uint32_t sr = 0, data_sz = 0; + long data_off = -1; + while (!feof(f)) { + char cid[4]; uint32_t csz; + if (fread(cid, 1, 4, f) != 4) break; + if (fread(&csz, 4, 1, f) != 1) break; + if (memcmp(cid, "fmt ", 4) == 0) { + std::vector buf(csz); fread(buf.data(), 1, csz, f); + channels = *(uint16_t*)&buf[2]; sr = *(uint32_t*)&buf[4]; bps = *(uint16_t*)&buf[14]; + } else if (memcmp(cid, "data", 4) == 0) { + data_sz = csz; data_off = ftell(f); break; + } else { fseek(f, csz, SEEK_CUR); } + } + if (data_off < 0 || channels != 1 || bps != 16) { + fprintf(stderr, "WAV mono 16-bit attendu (ch=%d bps=%d)\n", channels, bps); + fclose(f); return {}; + } + sr_out = (int)sr; + fseek(f, data_off, SEEK_SET); + std::vector pcm(data_sz / 2); + fread(pcm.data(), 2, pcm.size(), f); + fclose(f); + return pcm; +} + +int main(int argc, char** argv) { + setvbuf(stderr, nullptr, _IONBF, 0); + setvbuf(stdout, nullptr, _IONBF, 0); + if (argc < 3) { + fprintf(stderr, "usage: %s [language=fr] [cpu|htp=htp]\n", + argv[0]); + return 1; + } + const char * model_dir = argv[1]; + const char * wav_path = argv[2]; + const char * lang = (argc >= 4) ? argv[3] : "fr"; + const bool use_htp = !(argc >= 5 && !strcmp(argv[4], "cpu")); + + int sr = 0; + auto pcm = read_wav_pcm16(wav_path, sr); + if (pcm.empty()) return 2; + fprintf(stderr, "wav : %zu samples @ %d Hz = %.2f s\n", + pcm.size(), sr, (double)pcm.size() / sr); + if (sr != 16000) { + fprintf(stderr, "ERROR: sr=%d != 16000 (Whisper requiert 16kHz)\n", sr); return 3; + } + + SttEngineLoadCfg lc; + lc.model_dir = model_dir; + lc.use_htp = use_htp; + lc.n_threads = 6; + auto * eng = stt_engine_load(lc); + if (!eng) { fprintf(stderr, "stt_engine_load FAIL\n"); return 4; } + + SttTranscribeCfg tc; + tc.pcm16 = pcm.data(); + tc.n_samples = (int)pcm.size(); + tc.sample_rate = 16000; + tc.language = lang; + tc.force_transcribe = true; + + auto R = stt_engine_transcribe(eng, tc); + if (R.err) { fprintf(stderr, "transcribe FAIL err=%d\n", R.err); stt_engine_free(eng); return 5; } + + printf("\n=== TRANSCRIPTION (%s, %s) ===\n", use_htp ? "NPU" : "CPU", lang); + printf("%s\n", R.text.c_str()); + printf("\n=== TIMING ===\n"); + printf("mel %4d ms\n", R.mel_ms); + printf("encoder %4d ms\n", R.encoder_ms); + printf("decoder %4d ms (%d tokens)\n", R.decoder_ms, R.n_tokens); + printf("total %4d ms for %.2f s audio => RTF %.3f\n", + R.total_ms, (double)pcm.size() / sr, R.rtf); + + stt_engine_free(eng); + return 0; +} diff --git a/dist/jni/stt_engine.cpp b/dist/jni/stt_engine.cpp index 41c7a56..176ec2b 100644 --- a/dist/jni/stt_engine.cpp +++ b/dist/jni/stt_engine.cpp @@ -1,65 +1,112 @@ // Implementation stt_engine — voir stt_engine.h pour l'API publique. // -// S1.3 (cette session) : squelette compilable, VAD complet, mel via kazeia_mel. -// stt_engine_load / transcribe / free : stubs renvoyant err=-99 jusqu'au port ORT -// QNN en S2. +// Port C++ complet de WhisperHybridEngine.kt (527L Kotlin -> ce fichier). +// Backend : ONNX Runtime 1.24.3 + QNN ExecutionProvider (HTP V79 sur SM8750). +// Modèle : Qualcomm AI Hub HfWhisper KV-cache decoder. +// +// Pipeline transcribe : +// 1. PCM16 -> mel via kazeia_mel (cfg Whisper) -> [1, 80, 3000] fp16 +// 2. Encoder (NPU) : input_features -> N cross KV caches fp16 +// 3. Decoder (NPU) autoregressif, KV-cache, mask R-to-L, 200 steps max : +// input_ids[1,1] + attention_mask[1,1,1,200] + N self KV + N cross KV + position_ids +// -> logits[1,51865,1,1] fp16 + N updated self KV +// 4. argmax logits FP16 (override <|translate|> -> <|transcribe|>, EOT -> stop) +// 5. Token decode via BPE byte-level (vocab.json) -> texte UTF-8 +// +// Auto-detect dimensions (num_decoder_layers, num_decoder_heads) via session outputs. +// Bench prod ref : Whisper-Small FR 1.6s -> 825 ms total (mel 189 + enc 125 + dec 510), RTF 0.51. #include "stt_engine.h" #include "kazeia_mel.h" +#include "onnxruntime/onnxruntime_cxx_api.h" + #include #include +#include #include #include -#include +#include +#include +#include +#include #include +#include +#include // ============================================================================ -// VAD RMS énergie (port C++ de VadStage.kt) +// FP16 helpers (cohabite avec Ort::Float16_t qui est juste un wrapper uint16_t) +// ============================================================================ + +// IEEE 754 binary16 conversion (round-to-nearest-even). +static uint16_t fp32_to_fp16(float f) { + uint32_t x; std::memcpy(&x, &f, 4); + uint32_t sign = (x >> 16) & 0x8000u; + int32_t exp = (int32_t)((x >> 23) & 0xFFu) - 127 + 15; + uint32_t mant = x & 0x7FFFFFu; + if (exp >= 31) { // inf / nan + return (uint16_t)(sign | 0x7C00u | (mant ? 1u : 0u)); + } else if (exp <= 0) { // subnormal / underflow + if (exp < -10) return (uint16_t)sign; + mant |= 0x800000u; + uint32_t shift = (uint32_t)(14 - exp); + uint32_t round = (mant >> (shift - 1)) & 1u; + return (uint16_t)(sign | ((mant >> shift) + round)); + } else { + uint32_t round = (mant >> 12) & 1u; + return (uint16_t)(sign | (uint32_t)(exp << 10) | (mant >> 13) + round); + } +} + +static float fp16_to_fp32(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000u) << 16; + uint32_t exp = (h >> 10) & 0x1Fu; + uint32_t mant = h & 0x3FFu; + uint32_t out; + if (exp == 0) { + if (mant == 0) { out = sign; } + else { + // subnormal + int e = -1; + while (!(mant & 0x400u)) { mant <<= 1; e--; } + mant &= 0x3FFu; + out = sign | (uint32_t)((127 + e) << 23) | (mant << 13); + } + } else if (exp == 31) { + out = sign | 0x7F800000u | (mant << 13); + } else { + out = sign | (uint32_t)((127 - 15 + (int)exp) << 23) | (mant << 13); + } + float f; std::memcpy(&f, &out, 4); + return f; +} + +// ============================================================================ +// VAD RMS énergie (port C++ de VadStage.kt) -- identique à S1.3 // ============================================================================ -// -// État interne : compteur de frames parole / silence consécutifs, sliding sur -// chunks PCM. L'appelant push des chunks de taille frame_size; chaque push fait -// avancer l'état. is_speech / is_end_of_speech consultent l'état. struct SttVadState { - int sample_rate; - int frame_size; // 1600 = 100 ms @ 16 kHz - int rms_threshold; // 150 sur PCM16 brut (prod KazeiaService) - int min_speech_frames; // 3 -> 300 ms de parole pour déclencher - int silence_end_frames; // 8 -> 800 ms de silence pour finir - - int speech_count = 0; - int silence_count = 0; - bool in_speech = false; - bool end_of_speech = false; + int sample_rate, frame_size, rms_threshold, min_speech_frames, silence_end_frames; + int speech_count = 0, silence_count = 0; + bool in_speech = false, end_of_speech = false; }; SttVadState * stt_vad_new(int sample_rate, int frame_size, int rms_threshold, int min_speech_frames, int silence_end_frames) { auto * s = new SttVadState(); - s->sample_rate = sample_rate; - s->frame_size = frame_size; - s->rms_threshold = rms_threshold; - s->min_speech_frames = min_speech_frames; - s->silence_end_frames = silence_end_frames; + s->sample_rate = sample_rate; s->frame_size = frame_size; + s->rms_threshold = rms_threshold; + s->min_speech_frames = min_speech_frames; s->silence_end_frames = silence_end_frames; return s; } - void stt_vad_reset(SttVadState * s) { if (!s) return; - s->speech_count = 0; - s->silence_count = 0; - s->in_speech = false; - s->end_of_speech = false; + s->speech_count = 0; s->silence_count = 0; + s->in_speech = false; s->end_of_speech = false; } - void stt_vad_free(SttVadState * s) { delete s; } - bool stt_vad_is_speech(const SttVadState * s) { return s && s->in_speech; } bool stt_vad_is_end_of_speech(const SttVadState * s) { return s && s->end_of_speech; } -// Calcule RMS d'un buffer PCM16, renvoie l'énergie en unités PCM brutes -// (compatible seuil 150 prod). static float rms_pcm16(const int16_t * pcm, int n) { if (n <= 0) return 0.0f; double acc = 0.0; @@ -70,29 +117,18 @@ static float rms_pcm16(const int16_t * pcm, int n) { bool stt_vad_push(SttVadState * s, const int16_t * pcm, int n_samples) { if (!s || !pcm || n_samples <= 0) return false; s->end_of_speech = false; - - // Traite par chunks de frame_size. Si n_samples < frame_size, on accumulera - // mentalement mais ici on calcule simplement la RMS sur ce qu'on a (l'app - // appelante pousse des chunks pleins de 100 ms en pratique). int offset = 0; while (offset + s->frame_size <= n_samples) { float r = rms_pcm16(pcm + offset, s->frame_size); if (r >= (float)s->rms_threshold) { - s->speech_count += 1; - s->silence_count = 0; - if (!s->in_speech && s->speech_count >= s->min_speech_frames) { - s->in_speech = true; - } + s->speech_count += 1; s->silence_count = 0; + if (!s->in_speech && s->speech_count >= s->min_speech_frames) s->in_speech = true; } else { s->silence_count += 1; - // Pas de reset speech_count tant qu'on est in_speech, c'est l'enchainement - // silence consécutifs qui clôt. if (s->in_speech && s->silence_count >= s->silence_end_frames) { - s->in_speech = false; - s->end_of_speech = true; - s->speech_count = 0; + s->in_speech = false; s->end_of_speech = true; s->speech_count = 0; } else if (!s->in_speech) { - s->speech_count = 0; // reset compteur si on n'a pas atteint le seuil + s->speech_count = 0; } } offset += s->frame_size; @@ -101,67 +137,623 @@ bool stt_vad_push(SttVadState * s, const int16_t * pcm, int n_samples) { } // ============================================================================ -// SttEngine (stubs — port ORT QNN en S2) +// JSON minimaliste (mel_filters.json = flat float array, vocab.json = {"k":v,...}) // ============================================================================ +// Parse mel_filters.json : "[1.234e-05, 0.0, ...]" (flat array de N floats). +static std::vector parse_mel_filters_json(const char * path) { + std::ifstream f(path); if (!f) return {}; + std::stringstream ss; ss << f.rdbuf(); + std::string s = ss.str(); + std::vector out; + out.reserve(80 * 201); + size_t i = 0; + while (i < s.size()) { + // Trouve prochain digit ou signe + while (i < s.size() && !(std::isdigit((unsigned char)s[i]) || s[i] == '-' || s[i] == '+' || s[i] == '.')) ++i; + if (i >= s.size()) break; + size_t j = i; + // Avance tant que c'est un caractère numérique + while (j < s.size() && (std::isdigit((unsigned char)s[j]) || s[j] == '.' || s[j] == 'e' || s[j] == 'E' || s[j] == '+' || s[j] == '-')) { + // garde-fou : - et + uniquement au début ou après e + if ((s[j] == '-' || s[j] == '+') && j != i && s[j-1] != 'e' && s[j-1] != 'E') break; + ++j; + } + if (j > i) { + try { out.push_back(std::stof(s.substr(i, j - i))); } catch (...) {} + } + i = j; + } + return out; +} + +// Parse vocab.json (HF whisper format) : {"!":0,"\"":1,...,"<|endoftext|>":50257,...} +// On retourne id -> token (chaîne UTF-8 brute, qui contient parfois des unicodes BPE). +static bool parse_vocab_json(const char * path, + std::unordered_map & id_to_token) { + std::ifstream f(path); if (!f) return false; + std::stringstream ss; ss << f.rdbuf(); + std::string s = ss.str(); + // On parcourt en cherchant des paires "key":number + size_t i = 0; + while (i < s.size()) { + // trouve " + while (i < s.size() && s[i] != '"') ++i; + if (i >= s.size()) break; + // Parse string key (handle \\, \", \uXXXX) + ++i; + std::string key; + while (i < s.size() && s[i] != '"') { + if (s[i] == '\\' && i + 1 < s.size()) { + char c = s[i+1]; + if (c == '"') { key.push_back('"'); i += 2; } + else if (c == '\\') { key.push_back('\\'); i += 2; } + else if (c == '/') { key.push_back('/'); i += 2; } + else if (c == 'n') { key.push_back('\n'); i += 2; } + else if (c == 't') { key.push_back('\t'); i += 2; } + else if (c == 'r') { key.push_back('\r'); i += 2; } + else if (c == 'b') { key.push_back('\b'); i += 2; } + else if (c == 'f') { key.push_back('\f'); i += 2; } + else if (c == 'u' && i + 5 < s.size()) { + // \uXXXX -> codepoint -> UTF-8 + unsigned cp = 0; + for (int k = 0; k < 4; ++k) { + char h = s[i+2+k]; + cp <<= 4; + if (h >= '0' && h <= '9') cp |= (unsigned)(h - '0'); + else if (h >= 'a' && h <= 'f') cp |= (unsigned)(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') cp |= (unsigned)(h - 'A' + 10); + } + // Encode UTF-8 + if (cp < 0x80) key.push_back((char)cp); + else if (cp < 0x800) { + key.push_back((char)(0xC0 | (cp >> 6))); + key.push_back((char)(0x80 | (cp & 0x3F))); + } else { + key.push_back((char)(0xE0 | (cp >> 12))); + key.push_back((char)(0x80 | ((cp >> 6) & 0x3F))); + key.push_back((char)(0x80 | (cp & 0x3F))); + } + i += 6; + } else { key.push_back(s[i]); ++i; } + } else { + key.push_back(s[i]); ++i; + } + } + if (i >= s.size()) break; + ++i; // skip closing " + // skip whitespace + ':' + whitespace + while (i < s.size() && (s[i] == ' ' || s[i] == ':' || s[i] == '\t' || s[i] == '\n')) ++i; + // Parse integer value + size_t j = i; + while (j < s.size() && std::isdigit((unsigned char)s[j])) ++j; + if (j > i) { + int id = std::stoi(s.substr(i, j - i)); + id_to_token[id] = key; + } + i = j; + // skip jusqu'à la prochaine "," + while (i < s.size() && s[i] != ',' && s[i] != '}') ++i; + if (i < s.size() && s[i] == '}') break; + } + return !id_to_token.empty(); +} + +// BPE byte-level GPT-2/Whisper : map character code (in vocab token strings) -> raw byte +static std::unordered_map build_bpe_unicode_to_byte() { + std::unordered_map map; + for (int b = 33; b <= 126; ++b) map[b] = b; + for (int b = 161; b <= 172; ++b) map[b] = b; + for (int b = 174; b <= 255; ++b) map[b] = b; + int n = 0; + for (int b = 0; b <= 255; ++b) { + bool found = false; + for (auto & kv : map) { if (kv.second == b) { found = true; break; } } + if (!found) { map[256 + n] = b; ++n; } + } + return map; +} + +// Parse UTF-8 codepoint at position i. Renvoie cp et avance i. +static unsigned utf8_next(const std::string & s, size_t & i) { + unsigned char c = (unsigned char)s[i++]; + if (c < 0x80) return c; + if ((c >> 5) == 0x6) { + unsigned cp = (c & 0x1F); + cp = (cp << 6) | ((unsigned char)s[i++] & 0x3F); + return cp; + } + if ((c >> 4) == 0xE) { + unsigned cp = (c & 0x0F); + cp = (cp << 6) | ((unsigned char)s[i++] & 0x3F); + cp = (cp << 6) | ((unsigned char)s[i++] & 0x3F); + return cp; + } + if ((c >> 3) == 0x1E) { + unsigned cp = (c & 0x07); + cp = (cp << 6) | ((unsigned char)s[i++] & 0x3F); + cp = (cp << 6) | ((unsigned char)s[i++] & 0x3F); + cp = (cp << 6) | ((unsigned char)s[i++] & 0x3F); + return cp; + } + return c; +} + +// Décode une liste de token ids -> texte UTF-8 via BPE byte-level Whisper. +// Filtre les tokens spéciaux <|...|>. +static std::string decode_tokens(const std::vector & ids, + const std::unordered_map & vocab, + const std::unordered_map & b2u_byte_map) { + std::string bytes; + bytes.reserve(ids.size() * 2); + for (int id : ids) { + auto it = vocab.find(id); if (it == vocab.end()) continue; + const std::string & w = it->second; + if (w.size() >= 4 && w[0] == '<' && w[1] == '|' && + w[w.size()-2] == '|' && w[w.size()-1] == '>') continue; + size_t i = 0; + while (i < w.size()) { + unsigned cp = utf8_next(w, i); + auto bit = b2u_byte_map.find((int)cp); + if (bit != b2u_byte_map.end()) bytes.push_back((char)bit->second); + } + } + return bytes; +} + +// ============================================================================ +// SttEngine — ORT QNN HfWhisper KV-cache +// ============================================================================ + +namespace { +// Constantes Whisper (multilingue) — confirmées dans le code prod Kotlin. +constexpr int SOT = 50258; +constexpr int EOT = 50257; +constexpr int TRANSLATE_TOK = 50358; +constexpr int TRANSCRIBE_TOK = 50359; +constexpr int VOCAB_SIZE = 51865; +constexpr int MEAN_DECODE_LEN = 200; +constexpr int HEAD_DIM = 64; +constexpr float MASK_NEG = -100.0f; + +double now_s() { + using clk = std::chrono::steady_clock; + return std::chrono::duration(clk::now().time_since_epoch()).count(); +} +} // namespace + struct SttEngine { SttEngineLoadCfg cfg; - std::vector mel_basis; // [80, 201] librosa whisper - // S2 : - // void * encoder_session; // Ort::Session* - // void * decoder_session; - // std::vector vocab_ids; - // std::unordered_map token_to_id; + std::vector mel_basis; + + std::unique_ptr ort_env; + std::unique_ptr enc_opts, dec_opts; + std::unique_ptr enc_sess, dec_sess; + std::unique_ptr mem_info; + std::unique_ptr allocator; + + // Auto-detected dims (Whisper-Base: 6/8, Small: 12/12, Medium: 24/16) + int num_decoder_layers = 0; + int num_decoder_heads = 0; + + // Cached I/O names (for the decoder loop tight allocation budget) + std::vector enc_in_names_owned; + std::vector enc_in_names; + std::vector enc_out_names_owned; + std::vector enc_out_names; + + std::vector dec_in_names_owned; + std::vector dec_in_names; + std::vector dec_out_names_owned; + std::vector dec_out_names; + + // Vocab + BPE + std::unordered_map vocab; + std::unordered_map bpe_unicode_to_byte; + bool loaded = false; }; +// Charge un fichier binaire ou JSON pour mel filters et retourne le tableau. +// Tente d'abord mel_filters.bin, sinon mel_filters.json. +static bool load_mel_basis_any(const std::string & model_dir, + const kazeia_mel::MelConfig & cfg, + std::vector & out) { + if (kazeia_mel::load_mel_basis((model_dir + "/mel_filters.bin").c_str(), cfg, out)) return true; + auto v = parse_mel_filters_json((model_dir + "/mel_filters.json").c_str()); + const size_t expected = (size_t)cfg.n_mels * (cfg.n_fft/2+1); + if (v.size() == expected) { out = std::move(v); return true; } + fprintf(stderr, "stt_engine: mel basis introuvable (ni .bin ni .json valide) à %s\n", + model_dir.c_str()); + return false; +} + +// Détecte num_decoder_layers (en comptant k_cache_cross_*) et num_decoder_heads (shape[0] du 1er). +static void detect_dims(SttEngine & e) { + Ort::AllocatorWithDefaultOptions a; + size_t n_out = e.enc_sess->GetOutputCount(); + int n_layers = 0; + int n_heads = 0; + for (size_t i = 0; i < n_out; ++i) { + auto name_ptr = e.enc_sess->GetOutputNameAllocated(i, a); + std::string name = name_ptr.get(); + if (name.rfind("k_cache_cross_", 0) == 0) { + ++n_layers; + if (n_heads == 0) { + auto info = e.enc_sess->GetOutputTypeInfo(i); + auto tinfo = info.GetTensorTypeAndShapeInfo(); + auto shape = tinfo.GetShape(); + if (!shape.empty() && shape[0] > 0) n_heads = (int)shape[0]; + } + } + } + // Défauts si détection ratée (Whisper-Base = 6/8, Small = 12/12) + if (n_layers == 0) n_layers = 12; + if (n_heads == 0) n_heads = 12; + e.num_decoder_layers = n_layers; + e.num_decoder_heads = n_heads; +} + +// Cache les input/output names (les sessions ORT exigent const char**) +static void cache_io_names(Ort::Session & sess, + std::vector & in_owned, + std::vector & in_ptrs, + std::vector & out_owned, + std::vector & out_ptrs) { + Ort::AllocatorWithDefaultOptions a; + size_t ni = sess.GetInputCount(), no = sess.GetOutputCount(); + in_owned.reserve(ni); in_ptrs.reserve(ni); + for (size_t i = 0; i < ni; ++i) { + auto p = sess.GetInputNameAllocated(i, a); + in_owned.emplace_back(p.get()); + } + for (auto & s : in_owned) in_ptrs.push_back(s.c_str()); + out_owned.reserve(no); out_ptrs.reserve(no); + for (size_t i = 0; i < no; ++i) { + auto p = sess.GetOutputNameAllocated(i, a); + out_owned.emplace_back(p.get()); + } + for (auto & s : out_owned) out_ptrs.push_back(s.c_str()); +} + SttEngine * stt_engine_load(const SttEngineLoadCfg & cfg) { if (!cfg.model_dir) { fprintf(stderr, "stt_engine_load: model_dir requis\n"); return nullptr; } auto * eng = new SttEngine(); eng->cfg = cfg; - // Tente charger mel_filters.bin (préféré) ou mel_filters.json (sera porté en S2). - std::string mel_path = std::string(cfg.model_dir) + "/mel_filters.bin"; + std::string D = cfg.model_dir; + + // 1) Mel basis (binaire optimal, JSON fallback) auto mel_cfg = kazeia_mel::config_whisper(); - if (!kazeia_mel::load_mel_basis(mel_path.c_str(), mel_cfg, eng->mel_basis)) { - fprintf(stderr, "stt_engine_load: mel_filters.bin absent ou mauvaise taille à %s\n", - mel_path.c_str()); - // Pas fatal : à S2 on chargera depuis mel_filters.json. Pour S1.3 on continue - // pour permettre les tests VAD. + if (!load_mel_basis_any(D, mel_cfg, eng->mel_basis)) { + delete eng; return nullptr; } + + // 2) ORT env + try { + eng->ort_env = std::make_unique(ORT_LOGGING_LEVEL_WARNING, "kazeia_stt"); + eng->mem_info = std::make_unique( + Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemTypeDefault)); + eng->allocator = std::make_unique(); + } catch (const std::exception & e) { + fprintf(stderr, "stt_engine_load: Ort env FAIL: %s\n", e.what()); + delete eng; return nullptr; + } + + // 3) Sessions encoder + decoder. QNN EP si use_htp. + auto make_opts = [&](Ort::SessionOptions & opts) { + if (cfg.use_htp) { + // QNN ExecutionProvider via AppendExecutionProvider("QNN", {...}) + std::vector keys = { "backend_path" }; + std::vector vals = { "libQnnHtp.so" }; + opts.AppendExecutionProvider("QNN", + {{"backend_path", "libQnnHtp.so"}}); + } + opts.SetIntraOpNumThreads(cfg.n_threads); + }; + + std::string enc_path = D + "/HfWhisperEncoder.onnx"; + std::string dec_path = D + "/HfWhisperDecoder.onnx"; + + try { + eng->enc_opts = std::make_unique(); + make_opts(*eng->enc_opts); + double t0 = now_s(); + eng->enc_sess = std::make_unique(*eng->ort_env, enc_path.c_str(), *eng->enc_opts); + fprintf(stderr, "stt_engine_load: encoder loaded in %.0f ms\n", (now_s() - t0) * 1000.0); + + eng->dec_opts = std::make_unique(); + make_opts(*eng->dec_opts); + double t1 = now_s(); + eng->dec_sess = std::make_unique(*eng->ort_env, dec_path.c_str(), *eng->dec_opts); + fprintf(stderr, "stt_engine_load: decoder loaded in %.0f ms\n", (now_s() - t1) * 1000.0); + } catch (const std::exception & e) { + fprintf(stderr, "stt_engine_load: ORT session FAIL: %s\n", e.what()); + delete eng; return nullptr; + } + + // 4) Auto-détection des dims + cache des noms I/O + detect_dims(*eng); + cache_io_names(*eng->enc_sess, eng->enc_in_names_owned, eng->enc_in_names, + eng->enc_out_names_owned, eng->enc_out_names); + cache_io_names(*eng->dec_sess, eng->dec_in_names_owned, eng->dec_in_names, + eng->dec_out_names_owned, eng->dec_out_names); + fprintf(stderr, "stt_engine_load: %d layers / %d heads detected, enc I/O=%zu/%zu dec I/O=%zu/%zu\n", + eng->num_decoder_layers, eng->num_decoder_heads, + eng->enc_in_names.size(), eng->enc_out_names.size(), + eng->dec_in_names.size(), eng->dec_out_names.size()); + + // 5) Vocab.json + BPE byte map + if (!parse_vocab_json((D + "/vocab.json").c_str(), eng->vocab)) { + fprintf(stderr, "stt_engine_load: vocab.json absent ou parse FAIL à %s\n", D.c_str()); + delete eng; return nullptr; + } + eng->bpe_unicode_to_byte = build_bpe_unicode_to_byte(); + fprintf(stderr, "stt_engine_load: vocab %zu tokens, BPE map %zu entries\n", + eng->vocab.size(), eng->bpe_unicode_to_byte.size()); + eng->loaded = true; return eng; } +// Création tensor FP16 owned (heap buffer kept alive jusqu'à la fin de la frame). +// Whisper-decoder loop produit ~200 fwd × (1+1+2L+1) tensors, ~13MB/step au pic. OK CPU. +static Ort::Value make_fp16_tensor(const SttEngine & e, + const std::vector & data, + const std::vector & shape, + std::vector> & owner) { + owner.emplace_back(data); + auto & ref = owner.back(); + return Ort::Value::CreateTensor( + *e.mem_info, ref.data(), ref.size() * sizeof(uint16_t), + shape.data(), shape.size(), + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16); +} + +static Ort::Value make_int32_tensor(const SttEngine & e, int v, + const std::vector & shape, + std::vector> & owner) { + owner.push_back({v}); + auto & ref = owner.back(); + return Ort::Value::CreateTensor( + *e.mem_info, ref.data(), ref.size(), + shape.data(), shape.size()); +} + +// Trouve l'index argmax sur logits FP16 [VOCAB_SIZE]. +static int argmax_fp16_logits(const uint16_t * data, int n) { + int best = 0; float best_v = -1e30f; + for (int i = 0; i < n; ++i) { + float v = fp16_to_fp32(data[i]); + if (v > best_v) { best_v = v; best = i; } + } + return best; +} + +// Convertit FloatArray mel -> FP16 buffer (taille N_MELS * N_FRAMES = 80*3000 = 240000). +static std::vector mel_to_fp16(const std::vector & mel) { + std::vector out(mel.size()); + for (size_t i = 0; i < mel.size(); ++i) out[i] = fp32_to_fp16(mel[i]); + return out; +} + SttTranscribeResult stt_engine_transcribe(SttEngine * eng, const SttTranscribeCfg & cfg) { SttTranscribeResult R{}; if (!eng || !eng->loaded || !cfg.pcm16 || cfg.n_samples <= 0) { R.err = -1; return R; } - // Étape 1 : mel via kazeia_mel (Whisper config). Implémentée en S1.3 pour exercer - // la lib partagée ; le encoder/decoder ORT viennent en S2. + if (cfg.sample_rate != 16000) { + fprintf(stderr, "stt_engine_transcribe: sample_rate=%d ; 16000 attendu\n", cfg.sample_rate); + R.err = -10; return R; + } + double t_start = now_s(); + + // 1) PCM -> wav f32 std::vector wav((size_t)cfg.n_samples); for (int i = 0; i < cfg.n_samples; ++i) wav[i] = (float)cfg.pcm16[i] / 32768.0f; + const double audio_s = (double)cfg.n_samples / cfg.sample_rate; - if (eng->mel_basis.empty()) { - fprintf(stderr, "stt_engine_transcribe: mel_basis non chargé (mel_filters.bin manquant)\n"); - R.err = -2; return R; - } + // 2) Mel via kazeia_mel auto mel_cfg = kazeia_mel::config_whisper(); + double tm0 = now_s(); int T = 0; auto mel = kazeia_mel::compute(wav, eng->mel_basis, mel_cfg, T); if (mel.empty()) { R.err = -3; return R; } - R.mel_ms = 0; // TODO: timer + R.mel_ms = (int)((now_s() - tm0) * 1000.0); - // Étape 2..N : encoder NPU + decoder loop + tokenizer BPE -> S2. - fprintf(stderr, "stt_engine_transcribe: stubs (port S2 requis). mel %d frames OK.\n", T); - R.err = -99; // not implemented yet + // 3) Encoder — fp16 input -> N cross KV caches fp16 + double te0 = now_s(); + auto mel_fp16 = mel_to_fp16(mel); + std::vector> tensor_buffers; + tensor_buffers.reserve(eng->num_decoder_layers * 4 + 4); + std::vector mel_shape = {1, 80, 3000}; + + std::vector enc_inputs; + enc_inputs.emplace_back(make_fp16_tensor(*eng, mel_fp16, mel_shape, tensor_buffers)); + + // ORT expects: const char* const* input_names + Value* + size_t input_count. + std::vector enc_outputs; + try { + enc_outputs = eng->enc_sess->Run( + Ort::RunOptions{nullptr}, + eng->enc_in_names.data(), enc_inputs.data(), enc_inputs.size(), + eng->enc_out_names.data(), eng->enc_out_names.size()); + } catch (const std::exception & e) { + fprintf(stderr, "stt_engine_transcribe: encoder Run FAIL: %s\n", e.what()); + R.err = -4; return R; + } + R.encoder_ms = (int)((now_s() - te0) * 1000.0); + + // 4) Map outputs (encoder) by name -> get cross KV pointers + shapes + // Owned copies for the decoder loop (will be reused across all steps). + struct KvCache { + std::vector data; + std::vector shape; + }; + std::vector cross_k(eng->num_decoder_layers), cross_v(eng->num_decoder_layers); + for (size_t i = 0; i < eng->enc_out_names.size(); ++i) { + std::string nm = eng->enc_out_names_owned[i]; + if (nm.rfind("k_cache_cross_", 0) == 0) { + int idx = std::stoi(nm.substr(14)); + if (idx < eng->num_decoder_layers) { + auto & val = enc_outputs[i]; + auto shape = val.GetTensorTypeAndShapeInfo().GetShape(); + size_t n = 1; for (auto d : shape) n *= (size_t)d; + cross_k[idx].shape = shape; + cross_k[idx].data.resize(n); + std::memcpy(cross_k[idx].data.data(), val.GetTensorMutableData(), n * 2); + } + } else if (nm.rfind("v_cache_cross_", 0) == 0) { + int idx = std::stoi(nm.substr(14)); + if (idx < eng->num_decoder_layers) { + auto & val = enc_outputs[i]; + auto shape = val.GetTensorTypeAndShapeInfo().GetShape(); + size_t n = 1; for (auto d : shape) n *= (size_t)d; + cross_v[idx].shape = shape; + cross_v[idx].data.resize(n); + std::memcpy(cross_v[idx].data.data(), val.GetTensorMutableData(), n * 2); + } + } + } + enc_outputs.clear(); + + // 5) Decoder loop autoregressif KV-cache (port exact decodeHfKvCache.kt) + double td0 = now_s(); + const int kv_slots = MEAN_DECODE_LEN - 1; // 199 + // Self KV layout : k [H, 1, head_dim, slots] ; v [H, 1, slots, head_dim]. + const size_t self_k_n = (size_t)eng->num_decoder_heads * HEAD_DIM * kv_slots; + const size_t self_v_n = (size_t)eng->num_decoder_heads * kv_slots * HEAD_DIM; + const std::vector self_k_shape = {eng->num_decoder_heads, 1, HEAD_DIM, kv_slots}; + const std::vector self_v_shape = {eng->num_decoder_heads, 1, kv_slots, HEAD_DIM}; + + std::vector self_k(eng->num_decoder_layers), self_v(eng->num_decoder_layers); + for (int l = 0; l < eng->num_decoder_layers; ++l) { + self_k[l].shape = self_k_shape; self_k[l].data.assign(self_k_n, fp32_to_fp16(0.0f)); + self_v[l].shape = self_v_shape; self_v[l].data.assign(self_v_n, fp32_to_fp16(0.0f)); + } + + std::vector mask_f32((size_t)MEAN_DECODE_LEN, MASK_NEG); + std::vector mask_fp16((size_t)MEAN_DECODE_LEN); + + std::vector generated; + generated.reserve(MEAN_DECODE_LEN); + + int current_token = SOT; + int position_id = 0; + + // Préalloue les noms decoder (input_ids, attention_mask, k/v_cache_self_*_in, + // k/v_cache_cross_*, position_ids). On les chercher dans dec_in_names_owned. + // Cache des indices pour Run() : on passe les Value dans l'ordre des noms cachés. + + for (int step = 0; step < MEAN_DECODE_LEN - 1; ++step) { + // Mask R-to-L + mask_f32[MEAN_DECODE_LEN - step - 1] = 0.0f; + for (int i = 0; i < MEAN_DECODE_LEN; ++i) mask_fp16[i] = fp32_to_fp16(mask_f32[i]); + + // Build Values aligned avec dec_in_names_owned[i] ordering + tensor_buffers.clear(); + std::vector> int_buffers; + + std::vector dec_inputs; + dec_inputs.reserve(eng->dec_in_names.size()); + + for (size_t i = 0; i < eng->dec_in_names_owned.size(); ++i) { + const std::string & nm = eng->dec_in_names_owned[i]; + if (nm == "input_ids") { + dec_inputs.emplace_back(make_int32_tensor(*eng, current_token, {1, 1}, int_buffers)); + } else if (nm == "attention_mask") { + dec_inputs.emplace_back(make_fp16_tensor(*eng, mask_fp16, + {1, 1, 1, (int64_t)MEAN_DECODE_LEN}, tensor_buffers)); + } else if (nm == "position_ids") { + dec_inputs.emplace_back(make_int32_tensor(*eng, position_id, {1}, int_buffers)); + } else if (nm.rfind("k_cache_self_", 0) == 0 && nm.find("_in") != std::string::npos) { + int idx = std::stoi(nm.substr(13)); + dec_inputs.emplace_back(make_fp16_tensor(*eng, self_k[idx].data, self_k[idx].shape, tensor_buffers)); + } else if (nm.rfind("v_cache_self_", 0) == 0 && nm.find("_in") != std::string::npos) { + int idx = std::stoi(nm.substr(13)); + dec_inputs.emplace_back(make_fp16_tensor(*eng, self_v[idx].data, self_v[idx].shape, tensor_buffers)); + } else if (nm.rfind("k_cache_cross_", 0) == 0) { + int idx = std::stoi(nm.substr(14)); + dec_inputs.emplace_back(make_fp16_tensor(*eng, cross_k[idx].data, cross_k[idx].shape, tensor_buffers)); + } else if (nm.rfind("v_cache_cross_", 0) == 0) { + int idx = std::stoi(nm.substr(14)); + dec_inputs.emplace_back(make_fp16_tensor(*eng, cross_v[idx].data, cross_v[idx].shape, tensor_buffers)); + } else { + fprintf(stderr, "stt_engine_transcribe: input decoder inconnu : %s\n", nm.c_str()); + R.err = -5; return R; + } + } + + // Run decoder + std::vector dec_outputs; + try { + dec_outputs = eng->dec_sess->Run( + Ort::RunOptions{nullptr}, + eng->dec_in_names.data(), dec_inputs.data(), dec_inputs.size(), + eng->dec_out_names.data(), eng->dec_out_names.size()); + } catch (const std::exception & e) { + fprintf(stderr, "stt_engine_transcribe: decoder Run step=%d FAIL: %s\n", step, e.what()); + R.err = -6; return R; + } + + // Parse outputs : logits + updated self KV + int logits_idx = -1; + for (size_t i = 0; i < eng->dec_out_names_owned.size(); ++i) { + if (eng->dec_out_names_owned[i] == "logits") { logits_idx = (int)i; break; } + } + if (logits_idx < 0) { + fprintf(stderr, "stt_engine_transcribe: pas de sortie 'logits' au decoder\n"); + R.err = -7; return R; + } + const uint16_t * logits_data = dec_outputs[logits_idx].GetTensorMutableData(); + int token = argmax_fp16_logits(logits_data, VOCAB_SIZE); + + // Override translate -> transcribe + if (cfg.force_transcribe && token == TRANSLATE_TOK) token = TRANSCRIBE_TOK; + + // Update self KV from outputs + for (size_t i = 0; i < eng->dec_out_names_owned.size(); ++i) { + const std::string & nm = eng->dec_out_names_owned[i]; + if (nm.rfind("k_cache_self_", 0) == 0 && nm.find("_out") != std::string::npos) { + int idx = std::stoi(nm.substr(13)); + auto & val = dec_outputs[i]; + size_t n = 1; for (auto d : val.GetTensorTypeAndShapeInfo().GetShape()) n *= (size_t)d; + if (n == self_k[idx].data.size()) { + std::memcpy(self_k[idx].data.data(), val.GetTensorMutableData(), n * 2); + } + } else if (nm.rfind("v_cache_self_", 0) == 0 && nm.find("_out") != std::string::npos) { + int idx = std::stoi(nm.substr(13)); + auto & val = dec_outputs[i]; + size_t n = 1; for (auto d : val.GetTensorTypeAndShapeInfo().GetShape()) n *= (size_t)d; + if (n == self_v[idx].data.size()) { + std::memcpy(self_v[idx].data.data(), val.GetTensorMutableData(), n * 2); + } + } + } + dec_outputs.clear(); + + if (token == EOT) break; + if (token < 50257) generated.push_back(token); + + current_token = token; + position_id += 1; + } + R.decoder_ms = (int)((now_s() - td0) * 1000.0); + + // 6) Token decode -> texte + R.text = decode_tokens(generated, eng->vocab, eng->bpe_unicode_to_byte); + R.n_tokens = (int)generated.size(); R.detected_language = cfg.language; + R.total_ms = (int)((now_s() - t_start) * 1000.0); + R.rtf = (audio_s > 0) ? (float)((double)R.total_ms / (audio_s * 1000.0)) : 0.0f; return R; } void stt_engine_free(SttEngine * eng) { if (!eng) return; + eng->enc_sess.reset(); eng->dec_sess.reset(); + eng->enc_opts.reset(); eng->dec_opts.reset(); + eng->ort_env.reset(); delete eng; } diff --git a/dist/lib/README_QNN.md b/dist/lib/README_QNN.md new file mode 100644 index 0000000..438c6ef --- /dev/null +++ b/dist/lib/README_QNN.md @@ -0,0 +1,19 @@ +# Libs QNN (Qualcomm Neural Network Runtime) + +Requises par `libkazeia_stt.so` pour le backend HTP V79 (Whisper-Small NPU). +Non versionnées dans git (98 MB total). Source : Maven `com.qualcomm.qti:qnn-runtime:2.44.0`. + +Pour récupérer (cache Gradle après un build d'app) : +```bash +QNN=~/.gradle/caches/8.12/transforms/*/transformed/qnn-runtime-2.44.0/jni/arm64-v8a +mkdir -p /opt/Kazeia-engine/dist/lib/qnn +cp $QNN/lib*.so /opt/Kazeia-engine/dist/lib/qnn/ +``` + +Côté app Android : déjà fournies par Gradle via `qnn-runtime:2.44.0`. Pas besoin de copier +manuellement dans `jniLibs/`. + +Pour le bench standalone (kazeia_stt_cli sur tablette adb), push : +```bash +adb push /opt/Kazeia-engine/dist/lib/qnn/*.so /data/local/tmp/stt/bin/ +```