From a41619ed67871dfe95e50cbb264cba1f75b67431 Mon Sep 17 00:00:00 2001 From: Kazeia Team Date: Tue, 14 Apr 2026 13:48:37 +0200 Subject: [PATCH] TTS: keep BigVGAN on CPU after GPU regression; LLM filter strips more tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #2 BigVGAN GPU experiment: ORT-QNN GPU EP loaded the v2_decoder_conv ONNX model successfully (session creation 463 ms, no fallback warnings) but per-phrase inference jumped to ~3.5 s vs ~2 s on CPU 8-thread. The GPU/CPU memory transfer cost dominates for this conv-heavy decoder, and the optimization went the wrong way. Comment block updated to record both the HTP and GPU paths as tried-and-rejected so future passes don't re-walk the same ground. LLM streaming filter: extend the lookahead-based … suppressor to also strip singleton special tokens (<|im_start|>, <|im_end|>, <|endoftext|>). Previously the closing <|im_end|> at end of the assistant's turn leaked into the SentenceStreamer and ended up as a spurious sentence at the end of the TTS output. Same lookahead-buffer trick handles split tokens. Validated end-to-end: 'Bonjour, comment vas-tu ?' → "Bonjour ! Je vais bien, merci. Comment vas-tu ?" → seg 0 "Bonjour !", seg 1 "Je vais bien, merci." (no <|im_end|>), BigVGAN back to 1.8 s/phrase. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../com/kazeia/llm/ExecuTorchLlmEngine.kt | 28 ++++++++++++++----- .../java/com/kazeia/tts/Qwen3TtsEngine.kt | 7 ++++- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt index 055772a..58127f0 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt @@ -115,29 +115,43 @@ class ExecuTorchLlmEngine( var inThink = false val tokenScan = StringBuilder() // small lookahead to spot tag boundaries + // Singleton special tokens that should never reach the TTS streamer + // (they leak when the model wraps its reply or signals end-of-turn). + val stripTokens = listOf("<|im_start|>", "<|im_end|>", "<|endoftext|>") + val maxTagLen = listOf("", "", "<|im_start|>", "<|im_end|>", "<|endoftext|>") + .maxOf { it.length } + val cb = object : LlmCallback { override fun onResult(result: String) { if (firstTokenMs < 0) firstTokenMs = System.currentTimeMillis() - startTime responseBuilder.append(result) - // Forward to caller only outside blocks. We accumulate - // a tiny lookahead buffer so tag tokens that arrive split - // ("") still match. + // Forward to caller only outside blocks, and strip + // singleton special tokens. We accumulate a tiny lookahead buffer + // so tag tokens that arrive split ("") still match. tokenScan.append(result) while (true) { if (!inThink) { val open = tokenScan.indexOf("") if (open < 0) { - // No tag pending — flush everything up to a safe point - // (length minus 7 for the longest tag we look for). - val safe = tokenScan.length - "".length + // No open pending — strip any singleton tokens + // that fully landed in the buffer, then flush prose + // up to a safe point preserving lookahead. + for (tok in stripTokens) { + var idx = tokenScan.indexOf(tok) + while (idx >= 0) { + tokenScan.delete(idx, idx + tok.length) + idx = tokenScan.indexOf(tok) + } + } + val safe = tokenScan.length - maxTagLen if (safe > 0) { onToken?.invoke(tokenScan.substring(0, safe)) tokenScan.delete(0, safe) } break } - // Flush the prose before the tag, then enter think mode. + // Flush the prose before the tag, then enter think mode. if (open > 0) onToken?.invoke(tokenScan.substring(0, open)) tokenScan.delete(0, open + "".length) inThink = true diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt index 461f10d..acf5a9f 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt @@ -243,7 +243,12 @@ class Qwen3TtsEngine( return session } - // Speech decoder V2 on CPU (HTP tested: BigVGAN convolutions too slow to compile) + // Speech decoder V2 on CPU. Two paths tried, both worse than CPU: + // - HTP: BigVGAN convolutions too slow to compile (timeout) + // - GPU Adreno via QNN GPU EP: model loads but per-phrase + // inference is ~3.5 s vs ~2 s on CPU (GPU/CPU memory transfer + // overhead dominates for this conv-heavy model) + // CPU 8-thread stays the practical optimum. val v2Path = "$path/v2_pre_conv" if (File("$v2Path/model.onnx").exists()) { nlog("Loading V2 speech decoder (CPU ONNX)...")