Kazeia
/
kazeia
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LLM: enable Qwen3-4B NPU (21 tok/s) in service pipeline 

- ExecuTorchLlmEngine: eval_mode 0 (our .pte is kv-mode, not hybrid)
- KazeiaService: call llm.load() after TTS init; try/catch falls back
  to echo mode if the runner or .pte are missing.

Pipeline on device: STT(WhisperHybridEngine) → [VoiceCommands → LLM] → TTS(Qwen3TtsEngine).
Validated on OnePlus Pad 3: LLM ready in ~8 s, gen 21.3 tok/s, RSS 1.76 GB in the
qnn_llama_runner subprocess (out-of-process from the Kazeia app).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kazeia Team 2026-04-13 23:00:25 +02:00
parent 19f934af25
commit 9930bfa392
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt
View File

@ -9,7 +9,8 @@ import java.io.File
/** /**
* LLM Engine using ExecuTorch + QNN backend via subprocess. * LLM Engine using ExecuTorch + QNN backend via subprocess.
* Calls qnn_llama_runner binary with root access. * Calls qnn_llama_runner binary with root access.
* Qwen3-0.6B at ~90 tok/s on NPU (Snapdragon 8 Elite). * Current tablet config: Qwen3-4B KV-mode, ~18-20 tok/s on Hexagon V79 (Snapdragon 8 Elite),
* TTFT 0.9 s, RSS 1.76 GB. Previously tested Qwen3-0.6B at ~76 tok/s.
*/ */
class ExecuTorchLlmEngine( class ExecuTorchLlmEngine(
private val onLog: ((String) -> Unit)? = null private val onLog: ((String) -> Unit)? = null
@ -179,7 +180,7 @@ if [ -n "${'$'}SYSTEM_ARGS" ]; then
--prompt "${'$'}PROMPT" \ --prompt "${'$'}PROMPT" \
--temperature ${'$'}TEMP \ --temperature ${'$'}TEMP \
--seq_len ${'$'}SEQ_LEN \ --seq_len ${'$'}SEQ_LEN \
--eval_mode 1 --eval_mode 0
else else
exec ./qnn_llama_runner \ exec ./qnn_llama_runner \
--model_path hybrid_llama_qnn.pte \ --model_path hybrid_llama_qnn.pte \
@ -191,7 +192,7 @@ else
--prompt "${'$'}PROMPT" \ --prompt "${'$'}PROMPT" \
--temperature ${'$'}TEMP \ --temperature ${'$'}TEMP \
--seq_len ${'$'}SEQ_LEN \ --seq_len ${'$'}SEQ_LEN \
--eval_mode 1 --eval_mode 0
fi fi
""".trimIndent() """.trimIndent()

13
kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt
View File

@ -516,10 +516,14 @@ class KazeiaService : Service() {
)) ))
} }
// LLM: disabled for debugging — echo mode // LLM: Qwen3-4B on Hexagon V79 via qnn_llama_runner.
_loadingState.value = LoadingState(50, "LLM (echo mode)…") _loadingState.value = LoadingState(50, "LLM Qwen3-4B NPU…")
llm = ExecuTorchLlmEngine { msg -> log(msg) } llm = ExecuTorchLlmEngine { msg -> log(msg) }
log("LLM: disabled for debug, echo mode active") try {
llm.load("${KazeiaApplication.MODELS_DIR}/qwen3-4b", com.kazeia.core.LlmConfig())
} catch (e: Exception) {
log("LLM load failed: ${e.message} — falling back to echo mode")
}
_loadingState.value = LoadingState(80, "Audio…") _loadingState.value = LoadingState(80, "Audio…")
// Audio // Audio
@ -539,7 +543,8 @@ class KazeiaService : Service() {
addMessage(ChatMessage( addMessage(ChatMessage(
role = ChatMessage.Role.KAZEIA, role = ChatMessage.Role.KAZEIA,
text = "Bonjour, je suis Kazeia. Mode perroquet actif." text = if (llm.isLoaded()) "Bonjour, je suis Kazeia."
else "Bonjour, je suis Kazeia. Mode perroquet actif."
)) ))
_pipelineState.value = PipelineState.Idle _pipelineState.value = PipelineState.Idle