diff --git a/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt index 055772a..58127f0 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/llm/ExecuTorchLlmEngine.kt @@ -115,29 +115,43 @@ class ExecuTorchLlmEngine( var inThink = false val tokenScan = StringBuilder() // small lookahead to spot tag boundaries + // Singleton special tokens that should never reach the TTS streamer + // (they leak when the model wraps its reply or signals end-of-turn). + val stripTokens = listOf("<|im_start|>", "<|im_end|>", "<|endoftext|>") + val maxTagLen = listOf("", "", "<|im_start|>", "<|im_end|>", "<|endoftext|>") + .maxOf { it.length } + val cb = object : LlmCallback { override fun onResult(result: String) { if (firstTokenMs < 0) firstTokenMs = System.currentTimeMillis() - startTime responseBuilder.append(result) - // Forward to caller only outside blocks. We accumulate - // a tiny lookahead buffer so tag tokens that arrive split - // ("") still match. + // Forward to caller only outside blocks, and strip + // singleton special tokens. We accumulate a tiny lookahead buffer + // so tag tokens that arrive split ("") still match. tokenScan.append(result) while (true) { if (!inThink) { val open = tokenScan.indexOf("") if (open < 0) { - // No tag pending — flush everything up to a safe point - // (length minus 7 for the longest tag we look for). - val safe = tokenScan.length - "".length + // No open pending — strip any singleton tokens + // that fully landed in the buffer, then flush prose + // up to a safe point preserving lookahead. + for (tok in stripTokens) { + var idx = tokenScan.indexOf(tok) + while (idx >= 0) { + tokenScan.delete(idx, idx + tok.length) + idx = tokenScan.indexOf(tok) + } + } + val safe = tokenScan.length - maxTagLen if (safe > 0) { onToken?.invoke(tokenScan.substring(0, safe)) tokenScan.delete(0, safe) } break } - // Flush the prose before the tag, then enter think mode. + // Flush the prose before the tag, then enter think mode. if (open > 0) onToken?.invoke(tokenScan.substring(0, open)) tokenScan.delete(0, open + "".length) inThink = true diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt index 461f10d..acf5a9f 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt @@ -243,7 +243,12 @@ class Qwen3TtsEngine( return session } - // Speech decoder V2 on CPU (HTP tested: BigVGAN convolutions too slow to compile) + // Speech decoder V2 on CPU. Two paths tried, both worse than CPU: + // - HTP: BigVGAN convolutions too slow to compile (timeout) + // - GPU Adreno via QNN GPU EP: model loads but per-phrase + // inference is ~3.5 s vs ~2 s on CPU (GPU/CPU memory transfer + // overhead dominates for this conv-heavy model) + // CPU 8-thread stays the practical optimum. val v2Path = "$path/v2_pre_conv" if (File("$v2Path/model.onnx").exists()) { nlog("Loading V2 speech decoder (CPU ONNX)...")