LLM: enable Qwen3-4B NPU (21 tok/s) in service pipeline
- ExecuTorchLlmEngine: eval_mode 0 (our .pte is kv-mode, not hybrid) - KazeiaService: call llm.load() after TTS init; try/catch falls back to echo mode if the runner or .pte are missing. Pipeline on device: STT(WhisperHybridEngine) → [VoiceCommands → LLM] → TTS(Qwen3TtsEngine). Validated on OnePlus Pad 3: LLM ready in ~8 s, gen 21.3 tok/s, RSS 1.76 GB in the qnn_llama_runner subprocess (out-of-process from the Kazeia app). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
19f934af25
commit
9930bfa392
|
|
@ -9,7 +9,8 @@ import java.io.File
|
|||
/**
|
||||
* LLM Engine using ExecuTorch + QNN backend via subprocess.
|
||||
* Calls qnn_llama_runner binary with root access.
|
||||
* Qwen3-0.6B at ~90 tok/s on NPU (Snapdragon 8 Elite).
|
||||
* Current tablet config: Qwen3-4B KV-mode, ~18-20 tok/s on Hexagon V79 (Snapdragon 8 Elite),
|
||||
* TTFT 0.9 s, RSS 1.76 GB. Previously tested Qwen3-0.6B at ~76 tok/s.
|
||||
*/
|
||||
class ExecuTorchLlmEngine(
|
||||
private val onLog: ((String) -> Unit)? = null
|
||||
|
|
@ -179,7 +180,7 @@ if [ -n "${'$'}SYSTEM_ARGS" ]; then
|
|||
--prompt "${'$'}PROMPT" \
|
||||
--temperature ${'$'}TEMP \
|
||||
--seq_len ${'$'}SEQ_LEN \
|
||||
--eval_mode 1
|
||||
--eval_mode 0
|
||||
else
|
||||
exec ./qnn_llama_runner \
|
||||
--model_path hybrid_llama_qnn.pte \
|
||||
|
|
@ -191,7 +192,7 @@ else
|
|||
--prompt "${'$'}PROMPT" \
|
||||
--temperature ${'$'}TEMP \
|
||||
--seq_len ${'$'}SEQ_LEN \
|
||||
--eval_mode 1
|
||||
--eval_mode 0
|
||||
fi
|
||||
""".trimIndent()
|
||||
|
||||
|
|
|
|||
|
|
@ -516,10 +516,14 @@ class KazeiaService : Service() {
|
|||
))
|
||||
}
|
||||
|
||||
// LLM: disabled for debugging — echo mode
|
||||
_loadingState.value = LoadingState(50, "LLM (echo mode)…")
|
||||
// LLM: Qwen3-4B on Hexagon V79 via qnn_llama_runner.
|
||||
_loadingState.value = LoadingState(50, "LLM Qwen3-4B NPU…")
|
||||
llm = ExecuTorchLlmEngine { msg -> log(msg) }
|
||||
log("LLM: disabled for debug, echo mode active")
|
||||
try {
|
||||
llm.load("${KazeiaApplication.MODELS_DIR}/qwen3-4b", com.kazeia.core.LlmConfig())
|
||||
} catch (e: Exception) {
|
||||
log("LLM load failed: ${e.message} — falling back to echo mode")
|
||||
}
|
||||
|
||||
_loadingState.value = LoadingState(80, "Audio…")
|
||||
// Audio
|
||||
|
|
@ -539,7 +543,8 @@ class KazeiaService : Service() {
|
|||
|
||||
addMessage(ChatMessage(
|
||||
role = ChatMessage.Role.KAZEIA,
|
||||
text = "Bonjour, je suis Kazeia. Mode perroquet actif."
|
||||
text = if (llm.isLoaded()) "Bonjour, je suis Kazeia."
|
||||
else "Bonjour, je suis Kazeia. Mode perroquet actif."
|
||||
))
|
||||
_pipelineState.value = PipelineState.Idle
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue