diff --git a/kazeia-android/app/build.gradle.kts b/kazeia-android/app/build.gradle.kts index 9c0c98b..a1ca3d9 100644 --- a/kazeia-android/app/build.gradle.kts +++ b/kazeia-android/app/build.gradle.kts @@ -44,8 +44,8 @@ android { applicationId = "com.kazeia" minSdk = 28 targetSdk = 36 - versionCode = 13 - versionName = "0.2.1" + versionCode = 14 + versionName = "0.2.2" // WebDAV de distribution — injecté depuis local.properties. buildConfigField("String", "WEBDAV_BASE", "\"${localProp("webdav.base")}\"") @@ -157,10 +157,9 @@ dependencies { implementation("com.qualcomm.qti:qnn-litert-delegate:2.44.0") implementation("com.qualcomm.qti:qnn-runtime:2.44.0") - // ExecuTorch JNI dependencies (for TTS CP on NPU) - implementation("com.facebook.fbjni:fbjni:0.7.0") - implementation("com.facebook.soloader:nativeloader:0.10.5") - implementation(files("libs/executorch.jar")) + // (ExecuTorch jar + fbjni + soloader retirés 2026-06-18 : seul le TTS legacy + // Qwen3 les utilisait, supprimé avec la bascule CosyVoice. Le LLM .pte passe + // par libkazeia_pte autonome, pas par org.pytorch.executorch.) // Tokeniseur HF (DJL) — pour PteLlmEngine (.pte NPU) : tokenise le ChatML Qwen // côté Kotlin puis passe les IDs au natif (tokeniseur natif de l'engine cassé pour Qwen). diff --git a/kazeia-android/app/src/main/AndroidManifest.xml b/kazeia-android/app/src/main/AndroidManifest.xml index 215984c..a779915 100644 --- a/kazeia-android/app/src/main/AndroidManifest.xml +++ b/kazeia-android/app/src/main/AndroidManifest.xml @@ -74,24 +74,6 @@ android:value="AI inference service for emotional support chatbot" /> - - - - - - - - - - /bench_tts.trigger`, relance app. - // Charge libkazeia_tts + 1 synthèse FR vers WAV. - val trigTts = java.io.File(getExternalFilesDir(null), "bench_tts.trigger") - if (trigTts.exists()) { - trigTts.delete() - Thread { com.kazeia.tts.BenchTtsHarness.runBench(this@KazeiaApplication) }.start() - } } private fun createNotificationChannel() { diff --git a/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt b/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt index 9a5e361..6228bf0 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt @@ -165,7 +165,7 @@ class ConfigStore private constructor(private val file: File) { ttsEnabled = true, sttEngine = "prod", llmEngine = "prod", - ttsEngine = "lib" // legacy "prod" = SIGSEGV avec le libllama fork ql embarqué + ttsEngine = "cosyvoice" // bascule 2026-06-18 : CosyVoice = seul TTS prod ) } diff --git a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaPipeline.kt b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaPipeline.kt index a80f977..4668db8 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaPipeline.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaPipeline.kt @@ -160,26 +160,9 @@ class KazeiaPipeline { val ttsEngine = tts ?: return _pipelineState.value = PipelineState.Speaking try { - val qwen = ttsEngine as? com.kazeia.tts.Qwen3TtsEngine - if (qwen != null) { - qwen.onSegmentPlaying = onSegmentPlaying - qwen.startStreamingSession() - val streamer = com.kazeia.tts.SentenceStreamer { raw -> - // Strip emoji / non-speakable pictographs before TTS - // so a standalone "😊" doesn't become its own noisy - // segment. The chat bubble keeps the original text — - // only the audio path sees the cleaned version. - val spoken = stripNonSpeakable(raw).trim() - if (spoken.isNotEmpty()) qwen.enqueueSentence(spoken) - } - streamer.append(text) - streamer.flush() - qwen.endStreamingSession() - } else { - ttsEngine.synthesizeAndPlay(text, context.language, - onComplete = { _pipelineState.value = PipelineState.Idle } - ) - } + // CosyVoice : lecture batch (synthèse N+1 pendant lecture N EN INTERNE). + ttsEngine.synthesizeAndPlay(text, context.language, + onComplete = { _pipelineState.value = PipelineState.Idle }) } catch (e: Exception) { log("TTS error: ${e.message}") } diff --git a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt index 3160d3d..dd498ec 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt @@ -277,290 +277,6 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think override fun onBind(intent: Intent?): IBinder = binder override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { - // Auto-pipeline trigger via: am startservice -n com.kazeia/.service.KazeiaService --ez run_pipeline true - if (intent?.getBooleanExtra("run_pipeline", false) == true) { - log("Auto-pipeline triggered via intent") - // Cancel any running pipeline to avoid blocking - currentPipelineJob?.cancel() - currentPipelineJob = serviceScope.launch(Dispatchers.IO) { - // Wait for TTS to be loaded - while (!::tts.isInitialized || tts !is com.kazeia.tts.Qwen3TtsEngine) { - kotlinx.coroutines.delay(500) - } - val embedsPath = "/data/local/tmp/kazeia/models/qwen3-tts-npu/full_pipeline_embeds.bin" - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine ?: return@launch - val audio = qwenTts.generateFromEmbeds(embedsPath) - log("Pipeline done: ${audio.size} samples (${audio.size/24000f}s)") - // Audio is played by the TTS engine internally - } - } - intent?.getStringExtra("stream_llm_mock")?.let { text -> - // Stage 3 validation path without a live LLM: feed the given text - // to the SentenceStreamer word-by-word with a short artificial - // inter-token pause so we can observe the "sentence N plays - // while sentence N+1 is generating" behaviour end-to-end. Used - // in debug builds where the LLM is disabled (echo mode). - // Optional companion extra `tts_voice` selects voice prefix file - // before generation (used by chantier ggml-cpu reference dump). - val voiceOverride = intent.getStringExtra("tts_voice") - log("Stream LLM mock: '${text.take(60)}${if (text.length>60) "..." else ""}'${voiceOverride?.let { " voice=$it" } ?: ""}") - serviceScope.launch { - try { - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine - if (qwenTts == null) { - // Backend non-Qwen3 (ex. KazeiaTtsEngine lib) : pas de - // streaming par phrase, on synthétise le texte complet - // via l'interface et on joue. Sert au test E2E du wiring. - if (!::tts.isInitialized) { log("Stream mock: TTS not ready"); return@launch } - _pipelineState.value = PipelineState.Speaking - val tS = System.currentTimeMillis() - tts.synthesizeAndPlay(text, "fr", - onComplete = { - log("Stream mock (lib) done at ${System.currentTimeMillis() - tS}ms") - _pipelineState.value = PipelineState.Idle - }) - return@launch - } - if (voiceOverride != null) { - qwenTts.setVoice(voiceOverride) - log("Voice set to: $voiceOverride") - } - qwenTts.startStreamingSession() - val tStart = System.currentTimeMillis() - var firstSentenceLogged = false - val streamer = com.kazeia.tts.SentenceStreamer { sentence -> - if (!firstSentenceLogged) { - log("First sentence at ${System.currentTimeMillis() - tStart}ms: '${sentence.take(60)}'") - firstSentenceLogged = true - } - qwenTts.enqueueSentence(sentence) - } - // Fake LLM: word by word, 50 ms per word. Real LLM on - // device emits ~15-25 tokens/s which is similar. - for (word in text.split(" ")) { - streamer.append("$word ") - kotlinx.coroutines.delay(50) - } - streamer.flush() - qwenTts.endStreamingSession() - log("Stream LLM mock done at ${System.currentTimeMillis() - tStart}ms") - } catch (e: Exception) { - log("Stream LLM mock error: ${e.message}") - e.printStackTrace() - } - } - } - intent?.getStringExtra("stream_llm")?.let { prompt -> - // Stage 3 end-to-end: LLM generates tokens → SentenceStreamer - // splits at terminal punctuation → each sentence is enqueued - // to the TTS streaming session → audio plays while the LLM - // keeps generating the next sentence. Removes the old "wait - // for full response, then start TTS" pattern which gave a - // long silence after the user stopped speaking. - log("Stream LLM: '${prompt.take(60)}${if (prompt.length>60) "..." else ""}'") - serviceScope.launch { - try { - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine ?: run { - log("Stream LLM: TTS is not Qwen3 engine, abort"); return@launch - } - if (!::llm.isInitialized || !llm.isLoaded()) { - log("Stream LLM: LLM not ready"); return@launch - } - // Set pipeline state to Speaking so the continuous- - // listening mic loop (line ~824) drops frames during - // TTS playback. Without this, the mic picks up the - // tablet speaker and feeds our own TTS back into STT, - // creating an infinite loop. - _pipelineState.value = PipelineState.Speaking - qwenTts.startStreamingSession() - val tStart = System.currentTimeMillis() - var firstSentenceLogged = false - val streamer = com.kazeia.tts.SentenceStreamer { sentence -> - if (!firstSentenceLogged) { - log("First sentence at ${System.currentTimeMillis() - tStart}ms: '${sentence.take(60)}'") - firstSentenceLogged = true - } - qwenTts.enqueueSentence(sentence) - } - val result = llm.generate( - prompt = prompt, - params = com.kazeia.core.SamplingParams(maxNewTokens = 120, temperature = 0.7f), - onToken = { token -> - streamer.append(token) - true // keep generating - } - ) - streamer.flush() - log("LLM done: ${result.tokenCount} tokens in ${result.timeMs}ms") - qwenTts.endStreamingSession() - log("Stream LLM total: ${System.currentTimeMillis() - tStart}ms") - } catch (e: Exception) { - log("Stream LLM error: ${e.message}") - e.printStackTrace() - } finally { - // Back to Idle so the next mic frame is accepted. - _pipelineState.value = PipelineState.Idle - } - } - } - intent?.getStringExtra("stream_text")?.let { text -> - // Stage 2 streaming from arbitrary text: BPE tokenize on-device, - // look up embeds in the full Qwen3 vocab, run the existing - // interleaved Hexagon generation loop, and play each segment - // as soon as it's decoded. No PC-side prep required. - log("Stream text: '${text.take(60)}${if (text.length>60) "..." else ""}'") - serviceScope.launch { - try { - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine ?: return@launch - val sr = 24000 - val track = android.media.AudioTrack.Builder() - .setAudioAttributes(android.media.AudioAttributes.Builder() - .setUsage(android.media.AudioAttributes.USAGE_MEDIA) - .setContentType(android.media.AudioAttributes.CONTENT_TYPE_SPEECH) - .build()) - .setAudioFormat(android.media.AudioFormat.Builder() - .setEncoding(android.media.AudioFormat.ENCODING_PCM_16BIT) - .setSampleRate(sr) - .setChannelMask(android.media.AudioFormat.CHANNEL_OUT_MONO) - .build()) - .setBufferSizeInBytes(sr * 4) - .setTransferMode(android.media.AudioTrack.MODE_STREAM) - .build() - track.play() - val tStart = System.currentTimeMillis() - var firstLogged = false - qwenTts.synthesizeTextStreaming(text) { segIdx, audio -> - if (!firstLogged) { - log("First audio out at ${System.currentTimeMillis() - tStart}ms (seg ${segIdx+1})") - firstLogged = true - } - track.write(audio, 0, audio.size) - } - track.stop(); track.release() - log("Stream text done at ${System.currentTimeMillis() - tStart}ms") - } catch (e: Exception) { - log("Stream text error: ${e.message}") - e.printStackTrace() - } - } - } - intent?.getStringExtra("stream_pipeline")?.let { embedsPath -> - // Stage 1 streaming pipeline: generate segment-by-segment and play each - // segment the moment it's ready via an AudioTrack MODE_STREAM. First audio - // arrives ~5s after request (time to generate 1 segment) instead of ~15s - // for the whole phrase. Per-segment WAVs + concatenated full WAV are - // written to /data/local/tmp/kazeia/kazeia_stream_seg*.wav and _full.wav - // by the engine itself — the service only handles playback. - log("Stream pipeline from pre-computed embeds: $embedsPath") - serviceScope.launch { - try { - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine ?: return@launch - val sr = 24000 - val track = android.media.AudioTrack.Builder() - .setAudioAttributes(android.media.AudioAttributes.Builder() - .setUsage(android.media.AudioAttributes.USAGE_MEDIA) - .setContentType(android.media.AudioAttributes.CONTENT_TYPE_SPEECH) - .build()) - .setAudioFormat(android.media.AudioFormat.Builder() - .setEncoding(android.media.AudioFormat.ENCODING_PCM_16BIT) - .setSampleRate(sr) - .setChannelMask(android.media.AudioFormat.CHANNEL_OUT_MONO) - .build()) - .setBufferSizeInBytes(sr * 4) // 2s mono pcm16 buffer, plenty for seg handoff - .setTransferMode(android.media.AudioTrack.MODE_STREAM) - .build() - track.play() - val tStart = System.currentTimeMillis() - var firstAudioLogged = false - qwenTts.generateFromEmbedsHexagonStreaming(embedsPath) { segIdx, audio -> - if (!firstAudioLogged) { - log("First audio out at ${System.currentTimeMillis() - tStart}ms (seg ${segIdx+1})") - firstAudioLogged = true - } - track.write(audio, 0, audio.size) - } - // Let AudioTrack drain the written samples before releasing. - track.stop() - track.release() - log("Stream pipeline done at ${System.currentTimeMillis() - tStart}ms") - } catch (e: Exception) { - log("Stream pipeline error: ${e.message}") - e.printStackTrace() - } - } - } - intent?.getStringExtra("full_pipeline")?.let { embedsPath -> - val savePath = intent.getStringExtra("save_wav") ?: "/data/local/tmp/kazeia/tts_output.wav" - log("Full pipeline from pre-computed embeds: $embedsPath") - serviceScope.launch { - try { - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine - if (qwenTts != null) { - val audio = qwenTts.generateFromEmbeds(embedsPath) - if (audio.isNotEmpty()) { - val f = java.io.File(savePath) - java.io.DataOutputStream(java.io.BufferedOutputStream(java.io.FileOutputStream(f))).use { out -> - val dataSize = audio.size * 2 - out.writeBytes("RIFF"); out.writeInt(Integer.reverseBytes(36 + dataSize)) - out.writeBytes("WAVEfmt ") - out.writeInt(Integer.reverseBytes(16)) - out.writeShort(java.lang.Short.reverseBytes(1).toInt()) - out.writeShort(java.lang.Short.reverseBytes(1).toInt()) - out.writeInt(Integer.reverseBytes(24000)) - out.writeInt(Integer.reverseBytes(48000)) - out.writeShort(java.lang.Short.reverseBytes(2).toInt()) - out.writeShort(java.lang.Short.reverseBytes(16).toInt()) - out.writeBytes("data"); out.writeInt(Integer.reverseBytes(dataSize)) - val bb = java.nio.ByteBuffer.allocate(dataSize).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (s in audio) bb.putShort(s) - out.write(bb.array()) - } - log("Full pipeline WAV: $savePath (${audio.size} samples, ${audio.size/24000f}s)") - } else { - log("Full pipeline: empty audio") - } - } - } catch (e: Exception) { - log("Full pipeline error: ${e.message}") - e.printStackTrace() - } - } - } - intent?.getStringExtra("decode_codes")?.let { codesPath -> - val savePath = intent.getStringExtra("save_wav") ?: "/data/local/tmp/kazeia/tts_output.wav" - val realTokens = intent.getIntExtra("real_tokens", 58) - log("Decoding precomputed codes: $codesPath → $savePath ($realTokens tokens)") - serviceScope.launch { - try { - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine - if (qwenTts != null) { - val audio = qwenTts.testWithPrecomputedCodes(codesPath, realTokens) - log("Decoded: ${audio.size} samples, ${audio.size * 1000L / 24000}ms") - // Write WAV - val f = java.io.File(savePath) - java.io.DataOutputStream(java.io.BufferedOutputStream(java.io.FileOutputStream(f))).use { out -> - val dataSize = audio.size * 2 - out.writeBytes("RIFF"); out.writeInt(Integer.reverseBytes(36 + dataSize)) - out.writeBytes("WAVEfmt ") - out.writeInt(Integer.reverseBytes(16)) - out.writeShort(java.lang.Short.reverseBytes(1).toInt()) - out.writeShort(java.lang.Short.reverseBytes(1).toInt()) - out.writeInt(Integer.reverseBytes(24000)) - out.writeInt(Integer.reverseBytes(48000)) - out.writeShort(java.lang.Short.reverseBytes(2).toInt()) - out.writeShort(java.lang.Short.reverseBytes(16).toInt()) - out.writeBytes("data"); out.writeInt(Integer.reverseBytes(dataSize)) - val bb = java.nio.ByteBuffer.allocate(dataSize).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (s in audio) bb.putShort(s) - out.write(bb.array()) - } - log("WAV saved: $savePath (${audio.size} samples)") - } - } catch (e: Exception) { - log("Decode error: ${e.message}") - } - } - } intent?.getStringExtra("tts_text")?.let { text -> val savePath = intent.getStringExtra("save_wav") if (savePath != null) { @@ -751,38 +467,15 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think val nativeLibDir = applicationInfo.nativeLibraryDir - // Dispatch backend TTS via ttsEngine. "prod" = Qwen3TtsEngine legacy - // (incompat ABI avec libllama fork-ql -> SIGSEGV sur install fraîche, - // cf feedback_tts_engine_default_lib). "lib" = KazeiaTtsEngine - // (libkazeia_tts CPU). "cosyvoice" = CosyVoiceTtsEngine (clonage vocal - // 9 langues, CPU-only) — NON validé device tant que la collision - // libggml n'est pas levée côté libs engine (cf CosyVoiceTtsEngine). - when (runtimeConfig.ttsEngine) { - "cosyvoice" -> { - _loadingState.value = LoadingState(15, "TTS CosyVoice…") - val cv = com.kazeia.tts.CosyVoiceTtsEngine(log = { msg -> log("[TTS] $msg") }) - cv.load() - if (!cv.isLoaded()) throw IllegalStateException("CosyVoice load failed") - tts = cv - log("TTS: CosyVoiceTtsEngine loaded (clonage vocal CPU-only)") - } - "lib" -> { - _loadingState.value = LoadingState(15, "TTS engine…") - val libTts = com.kazeia.tts.KazeiaTtsEngine(this@KazeiaService) { msg -> log("[TTS] $msg") } - libTts.load() - if (!libTts.isLoaded()) throw IllegalStateException("KazeiaTtsEngine load failed") - tts = libTts - log("TTS: KazeiaTtsEngine loaded (libkazeia_tts CPU-only)") - } - else -> { - _loadingState.value = LoadingState(15, "TTS Qwen3…") - val qwenTts = com.kazeia.tts.Qwen3TtsEngine(nativeLibDir, this@KazeiaService) { msg -> log("[TTS] $msg") } - qwenTts.load("$modelsDir/qwen3-tts-npu") - if (!qwenTts.isLoaded()) throw IllegalStateException("Qwen3-TTS load failed (no fallback in frozen stack)") - tts = qwenTts - log("TTS: Qwen3-TTS loaded (Hexagon NPU + ggml-cpu)") - } - } + // TTS = CosyVoice (seul moteur depuis la bascule 2026-06-18 ; les + // moteurs legacy Qwen3-TTS/lib ont été retirés). Clonage vocal 9 + // langues, CPU-only, RTF ~1.0. + _loadingState.value = LoadingState(15, "TTS CosyVoice…") + val cv = com.kazeia.tts.CosyVoiceTtsEngine(log = { msg -> log("[TTS] $msg") }) + cv.load() + if (!cv.isLoaded()) throw IllegalStateException("CosyVoice load failed") + tts = cv + log("TTS: CosyVoiceTtsEngine loaded (clonage vocal CPU-only)") _loadingState.value = LoadingState(20, "TTS OK") // STT = Whisper-Small NPU (HfWhisper Qualcomm) — voie validée @@ -1009,8 +702,6 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think val wavPath = "${KazeiaApplication.MODELS_DIR}/../voix/$voiceId.wav" when (val e = tts) { is com.kazeia.tts.CosyVoiceTtsEngine -> { e.setVoice(voiceId); log("Voix (cosyvoice): $voiceId") } - is com.kazeia.tts.Qwen3TtsEngine -> { e.setVoice(wavPath); log("Voix (qwen3): $voiceId") } - is com.kazeia.tts.KazeiaTtsEngine -> { e.setVoice(wavPath); log("Voix (lib): $voiceId") } else -> log("setVoiceId: moteur TTS inconnu — voix '$voiceId' ignorée") } } @@ -1633,41 +1324,6 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think metrics.begin("text", text) } serviceScope.launch { - // Special test commands — cancel previous pipeline first - if (text.trim().lowercase().let { it.startsWith("pipeline") || it.startsWith("!pipeline") || it.startsWith("\\!pipeline") || it == "go" }) { - currentPipelineJob?.cancel() - currentPipelineJob = kotlin.coroutines.coroutineContext[kotlinx.coroutines.Job] - val embedsPath = "/data/local/tmp/kazeia/models/qwen3-tts-npu/full_pipeline_embeds.bin" - val wavPath = "/data/local/tmp/kazeia/tts_output.wav" - addMessage(ChatMessage(role = ChatMessage.Role.SYSTEM, text = "Running full pipeline...")) - try { - val qwenTts = tts as? com.kazeia.tts.Qwen3TtsEngine - val audio = qwenTts?.generateFromEmbeds(embedsPath) ?: ShortArray(0) - if (audio.isNotEmpty()) { - val f = java.io.File(wavPath) - java.io.DataOutputStream(java.io.BufferedOutputStream(java.io.FileOutputStream(f))).use { out -> - val dataSize = audio.size * 2 - out.writeBytes("RIFF"); out.writeInt(Integer.reverseBytes(36 + dataSize)) - out.writeBytes("WAVEfmt "); out.writeInt(Integer.reverseBytes(16)) - out.writeShort(java.lang.Short.reverseBytes(1).toInt()) - out.writeShort(java.lang.Short.reverseBytes(1).toInt()) - out.writeInt(Integer.reverseBytes(24000)); out.writeInt(Integer.reverseBytes(48000)) - out.writeShort(java.lang.Short.reverseBytes(2).toInt()) - out.writeShort(java.lang.Short.reverseBytes(16).toInt()) - out.writeBytes("data"); out.writeInt(Integer.reverseBytes(dataSize)) - val bb = java.nio.ByteBuffer.allocate(dataSize).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (s in audio) bb.putShort(s) - out.write(bb.array()) - } - addMessage(ChatMessage(role = ChatMessage.Role.SYSTEM, text = "WAV: $wavPath (${audio.size/24000f}s)")) - } else { - addMessage(ChatMessage(role = ChatMessage.Role.SYSTEM, text = "Pipeline: empty audio")) - } - } catch (e: Exception) { - addMessage(ChatMessage(role = ChatMessage.Role.SYSTEM, text = "Pipeline error: ${e.message}")) - } - return@launch - } // Check voice commands before processing if (handleVoiceCommand(text)) return@launch @@ -1806,19 +1462,10 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think } catch (e: Exception) { log("[RAG] retrieve échec: ${e.message}") } } - // === Opt TTS-B: the TTS streaming session opens BEFORE llm.generate, - // so the first complete sentence emitted by the LLM (on terminal - // punctuation) is enqueued for synthesis while the LLM is still - // decoding the rest of the response. Saves ~LLM-wall-time of TTFA - // when the first sentence finishes before the LLM does (common for - // short conversational answers). Non-Qwen TTS backends that lack - // the enqueueSentence plumbing fall through to the legacy post-LLM - // pipeline.speakText batch call. - val qwen = tts as? com.kazeia.tts.Qwen3TtsEngine - // NB CosyVoice : PAS de streaming LLM→TTS. LLM decode et synthèse CosyVoice - // sont tous deux CPU-bound (saturent les cœurs) ; les chevaucher étrangle le - // LLM (mesuré : 17→0,55 tok/s, tour 1,1s→41s). On laisse le LLM finir vite - // (session ~1,1s) PUIS synthesizeAndPlay (chemin legacy) qui streame déjà + // TTS = CosyVoice (CPU) : PAS de streaming LLM→TTS. LLM decode et synthèse + // CosyVoice sont tous deux CPU-bound (saturent les cœurs) ; les chevaucher + // étrangle le LLM (mesuré : 17→0,55 tok/s, tour 1,1s→41s). On laisse le LLM + // finir vite (session ~1,1s) PUIS pipeline.speakText (batch) qui streame déjà // phrase-par-phrase EN INTERNE (synthèse N+1 pendant lecture N, sans contention). // Bulle vide + signal Thinking sur le visualizer : la couleur de l'orbe // pulse pendant que le LLM réfléchit, plus de "..." textuels parasites. @@ -1884,18 +1531,10 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think // se remplit token par token dans onToken (voir plus bas). // Streaming LLM→TTS réservé à Qwen3 (NPU/Hexagon : pas de contention CPU avec // le LLM). CosyVoice (CPU) passe par le chemin batch (voir note plus haut). - val ttsStreamer: com.kazeia.tts.SentenceStreamer? = - if (qwen != null && !llmOnly) { - qwen.onSegmentPlaying = onSegPlay - qwen.startStreamingSession() - com.kazeia.tts.SentenceStreamer { raw -> - val sIdx = sentenceCounter.getAndIncrement() - val dt = System.currentTimeMillis() - llmStreamStart - val spoken = pipeline.stripNonSpeakable(raw).trim() - log("[LLM.sentence] #$sIdx +${dt}ms → '$spoken'") - if (spoken.isNotEmpty()) qwen.enqueueSentence(spoken) - } - } else null + // CosyVoice (CPU) : pas de streaming LLM→TTS (contention CPU). Le LLM décode + // vite, puis synthèse batch via pipeline.speakText (qui streame déjà + // phrase-par-phrase EN INTERNE). ttsStreamer reste donc null. + val ttsStreamer: com.kazeia.tts.SentenceStreamer? = null // In LLM_ONLY_MODE, kill the typing dots as soon as the first token // arrives so the bubble starts revealing text immediately. Outside @@ -1981,15 +1620,8 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think // onToken ; on finalise avec la réponse nettoyée. updateMessageText(bubble.id, responseText) metrics.end("llm_only") - } else if (ttsStreamer != null && qwen != null) { - // Streaming path: tail fragment (if any) + drain playback. - ttsStreamer.flush() - qwen.endStreamingSession() - revealJobs.forEach { try { it.join() } catch (_: Exception) {} } - updateMessageText(bubble.id, responseText) - metrics.end("ok") } else { - // Legacy batch path for non-Qwen TTS engines. + // CosyVoice : synthèse batch (speakText streame en interne). _pipelineState.value = PipelineState.Speaking pipeline.speakText(responseText) { sentence, durationMs, envelope, spectrogram -> onSegPlay(sentence, durationMs, envelope, spectrogram) @@ -2006,9 +1638,7 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think totalMs = result.timeMs ) } else { - // Empty response — close any open streaming session cleanly - // and drop the placeholder bubble. - try { qwen?.endStreamingSession() } catch (_: Exception) {} + // Empty response — drop the placeholder bubble. _messages.value = _messages.value.filter { it.id != bubble.id } metrics.end("empty_response") } @@ -2017,7 +1647,6 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think log("ERROR: LLM generation error: ${e.message}") metrics.mark("LLM.error", "msg=\"${e.message?.take(80) ?: ""}\"") metrics.end("error") - try { qwen?.endStreamingSession() } catch (_: Exception) {} _pipelineState.value = PipelineState.Error(e.message ?: "Erreur LLM") addMessage(ChatMessage( role = ChatMessage.Role.SYSTEM, diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/BenchTtsHarness.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/BenchTtsHarness.kt deleted file mode 100644 index 18aab5e..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/BenchTtsHarness.kt +++ /dev/null @@ -1,56 +0,0 @@ -// Harness TTS ADB-pilotable, mirror du LLM bench. -// Déclenchement : `touch /bench_tts.trigger`, relance app. -// Charge libkazeia_tts CPU-only + 1 synthèse FR → fichier WAV. -package com.kazeia.tts - -import android.content.Context -import android.util.Log - -object BenchTtsHarness { - private const val TAG = "BenchTts" - - fun runBench(ctx: Context) { - val talker = "/data/local/tmp/kz-engine/talker_f32.gguf" - val vocab = "/data/local/tmp/kz-engine/Qwen3-4B-Q4_0.gguf" - val dump = "/data/local/tmp/kz-engine/tts_dump" - val outWav = "${ctx.getExternalFilesDir(null)}/tts_bench_out.wav" - - Log.i(TAG, "=== load libkazeia_tts CPU-only ===") - val t0 = System.currentTimeMillis() - val jni = TtsJni() - val h = try { - jni.nativeLoad(talker, vocab, dump, /*useHtp=*/false, /*nThreads=*/6, /*cpUseCache=*/true) - } catch (e: Throwable) { - Log.e(TAG, "load THROW: ${e.message}"); return - } - if (h == 0L) { Log.e(TAG, "load FAIL (handle=0). talker=$talker vocab=$vocab dump=$dump"); return } - Log.i(TAG, "load OK in ${System.currentTimeMillis()-t0}ms, handle=$h") - - val text = "Bonsoir Pierre, comment vous sentez-vous aujourd'hui ?" - Log.i(TAG, "=== synthèse : '$text' → $outWav ===") - val ts = System.currentTimeMillis() - val r = try { - jni.nativeSynthesize( - h, text, outWav, - /*maxSteps=*/256, /*seed=*/42, - 0.9f, 50, 0.95f, 1.10f, // cp sampling - 0.9f, 50, 0.95f, 1.10f // talker sampling - ) - } catch (e: Throwable) { - Log.e(TAG, "synth THROW: ${e.message}"); jni.nativeFree(h); return - } - val wall = System.currentTimeMillis() - ts - // IntArray = [err, frames, total_ms, prefill_ms, talker_ms, cp_ms, decoder_ms] - if (r == null || r.size < 7) { Log.e(TAG, "synth bad result: ${r?.toList()}"); jni.nativeFree(h); return } - val err = r[0]; val frames = r[1]; val totalMs = r[2] - val prefillMs = r[3]; val talkerMs = r[4]; val cpMs = r[5]; val decoderMs = r[6] - val audioMs = frames * 80 // 12.5 fps => 80 ms/frame - val rtf = if (audioMs > 0) totalMs.toFloat() / audioMs else 0f - Log.i(TAG, "synth err=$err frames=$frames audio=${audioMs}ms total=${totalMs}ms (wall=${wall}ms)") - Log.i(TAG, " prefill=$prefillMs talker=$talkerMs cp=$cpMs decoder=$decoderMs RTF=${"%.2f".format(rtf)}") - Log.i(TAG, " WAV écrit : $outWav (taille=${java.io.File(outWav).length()} octets)") - - jni.nativeFree(h) - Log.i(TAG, "=== Bench end ===") - } -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/KazeiaTtsEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/KazeiaTtsEngine.kt deleted file mode 100644 index a20030f..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/KazeiaTtsEngine.kt +++ /dev/null @@ -1,199 +0,0 @@ -// Adapter core.TtsEngine -> libkazeia_tts (engine CPU-only unifié). -// Remplace Qwen3TtsEngine quand ttsEngine="lib". Synthèse vers WAV temp via -// TtsJni, lecture PCM, playback AudioTrack (MODE_STATIC, ColorOS-safe). -// -// Assets requis (résolus sous baseDir) : talker_f32.gguf, Qwen3-4B-Q4_0.gguf, -// tts_dump/ (text_embed.bin, tp_*, cp_*.gguf, qwen3tts_decoder.gguf, ...). -package com.kazeia.tts - -import android.content.Context -import android.media.AudioAttributes -import android.media.AudioFormat -import android.media.AudioTrack -import com.kazeia.core.TtsEngine -import com.kazeia.core.TtsResult -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlinx.coroutines.withContext -import java.io.File -import kotlin.coroutines.resume -import java.nio.ByteBuffer -import java.nio.ByteOrder - -class KazeiaTtsEngine( - private val context: Context, - private val log: (String) -> Unit = {} -) : TtsEngine { - - companion object { - private const val SR = 24000 - private const val MAX_STEPS = 256 - private const val SEED = 42 - // En dessous de ce |sample|, une frame de 80 ms est considérée silence. - private const val SILENCE_THRESHOLD = 200 - private const val STAGING_DIR = "/data/local/tmp/kz-engine" - } - - private val jni = TtsJni() - private var handle: Long = 0L - private var audioTrack: AudioTrack? = null - private val sampling = TtsSampling() - // Voix demandée (chemin WAV). Réappliquée après chaque load (le x_vector est - // mémorisé natif par handle, et le handle change au reload). - @Volatile private var pendingVoiceWav: String? = null - - private fun baseDir(): String { - val models = com.kazeia.KazeiaApplication.MODELS_DIR - return if (File("$models/tts_dump/qwen3tts_decoder.gguf").exists()) models else STAGING_DIR - } - - override suspend fun load(modelPath: String?, voiceId: String?) { - if (handle != 0L) return - withContext(Dispatchers.Default) { - val base = baseDir() - val talker = "$base/talker_f32.gguf" - val vocab = "$base/Qwen3-4B-Q4_0.gguf" - val dump = "$base/tts_dump" - log("load: base=$base") - val t0 = System.currentTimeMillis() - handle = jni.nativeLoad(talker, vocab, dump, /*useHtp=*/false, /*nThreads=*/6, /*cpUseCache=*/true) - if (handle == 0L) { - log("load FAIL (handle=0). talker=$talker vocab=$vocab dump=$dump") - error("KazeiaTtsEngine load failed (handle=0)") - } - log("load OK in ${System.currentTimeMillis() - t0}ms") - // Réapplique la voix demandée avant le load (sinon le x_vector natif, - // indexé par handle, est perdu au reload et on retombe sur damien). - pendingVoiceWav?.let { applyVoice(it) } - } - } - - override fun isLoaded(): Boolean = handle != 0L - - /** - * Hot-swap de voix "basé sur le WAV" : le natif dérive le x_vector du fichier - * WAV (downmix + resample 24k embarqués) via le speaker encoder, et l'utilise - * pour tous les synth suivants. [voiceWavPath] = chemin vers un .wav de voix - * (ex: .../voix/elodie.wav). Effet au prochain segment synthétisé. - */ - fun setVoice(voiceWavPath: String) { - pendingVoiceWav = voiceWavPath - if (handle != 0L) applyVoice(voiceWavPath) - } - - private fun applyVoice(voiceWavPath: String) { - if (!File(voiceWavPath).exists()) { - log("setVoice: WAV introuvable: $voiceWavPath — voix inchangée") - return - } - val ok = try { jni.nativeSetVoiceWav(handle, voiceWavPath) } catch (e: Exception) { - log("setVoice: nativeSetVoiceWav exception: ${e.message}"); false - } - log(if (ok) "Voix basculée sur ${File(voiceWavPath).nameWithoutExtension}" - else "setVoice: encodage échoué pour $voiceWavPath (speaker encoder absent ?) — voix inchangée") - } - - override suspend fun synthesize(text: String, language: String): TtsResult = - withContext(Dispatchers.Default) { - require(handle != 0L) { "KazeiaTtsEngine: not loaded" } - val outWav = File(context.cacheDir, "kz_tts_${System.nanoTime()}.wav") - val r = jni.nativeSynthesize( - handle, text, outWav.absolutePath, MAX_STEPS, SEED, - sampling.cpTemp, sampling.cpTopK, sampling.cpTopP, sampling.cpRepPenalty, - sampling.talkerTemp, sampling.talkerTopK, sampling.talkerTopP, sampling.talkerRepPenalty - ) - if (r == null || r.size < 7 || r[0] != 0) { - outWav.delete() - error("KazeiaTtsEngine.synthesize err=${r?.getOrNull(0)} text=\"$text\"") - } - log("synth frames=${r[1]} total=${r[2]}ms (talker=${r[4]} cp=${r[5]} dec=${r[6]})") - val pcm = readWavPcm(outWav) - outWav.delete() - val trimmed = trimTrailingSilence(pcm) - TtsResult( - audioData = trimmed, - sampleRate = SR, - durationMs = (trimmed.size * 1000L / SR) - ) - } - - override suspend fun synthesizeAndPlay( - text: String, - language: String, - onStart: (() -> Unit)?, - onComplete: (() -> Unit)? - ) { - val result = synthesize(text, language) - if (result.audioData.isEmpty()) { onComplete?.invoke(); return } - onStart?.invoke() - // Suspend jusqu'à la FIN réelle de la lecture (marker AudioTrack), pas - // seulement le démarrage. Sans ça, l'appelant (pipeline.speakText) - // repasse l'état Speaking→Idle dès track.play(), le micro se démute - // et capture le WAV qui joue encore → écho/boucle. - suspendCancellableCoroutine { cont -> - cont.invokeOnCancellation { stop() } - playAudio(result.audioData) { if (cont.isActive) cont.resume(Unit) } - } - onComplete?.invoke() - } - - private fun playAudio(audioData: ShortArray, onComplete: () -> Unit) { - stop() - val track = AudioTrack.Builder() - .setAudioAttributes(AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_MEDIA) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build()) - .setAudioFormat(AudioFormat.Builder() - .setSampleRate(SR) - .setEncoding(AudioFormat.ENCODING_PCM_16BIT) - .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) - .build()) - .setBufferSizeInBytes(audioData.size * 2) - .setTransferMode(AudioTrack.MODE_STATIC) - .build() - audioTrack = track - track.write(audioData, 0, audioData.size) - track.setNotificationMarkerPosition(audioData.size) - track.setPlaybackPositionUpdateListener(object : AudioTrack.OnPlaybackPositionUpdateListener { - override fun onMarkerReached(t: AudioTrack?) { onComplete() } - override fun onPeriodicNotification(t: AudioTrack?) {} - }) - track.play() - } - - private fun readWavPcm(wav: File): ShortArray { - val bytes = wav.readBytes() - if (bytes.size <= 44) return ShortArray(0) - // En-tête PCM canonique 44 octets (écrit par le JNI). - val dataBytes = bytes.size - 44 - val n = dataBytes / 2 - val bb = ByteBuffer.wrap(bytes, 44, dataBytes).order(ByteOrder.LITTLE_ENDIAN) - return ShortArray(n) { bb.short } - } - - // Coupe le silence de fin (le talker pad jusqu'à MAX_STEPS si EOS n'est pas - // émis -> sans ça la lecture/orbe traînerait plusieurs secondes de blanc). - private fun trimTrailingSilence(pcm: ShortArray): ShortArray { - var end = pcm.size - while (end > 0 && kotlin.math.abs(pcm[end - 1].toInt()) < SILENCE_THRESHOLD) end-- - if (end == pcm.size) return pcm - // Garde ~120 ms de queue pour ne pas couper sec. - val tail = (SR * 120 / 1000) - val keep = minOf(pcm.size, end + tail) - return pcm.copyOfRange(0, keep) - } - - override fun stop() { - audioTrack?.let { - try { it.pause(); it.flush(); it.stop() } catch (_: Exception) {} - try { it.release() } catch (_: Exception) {} - } - audioTrack = null - } - - override fun release() { - stop() - if (handle != 0L) { try { jni.nativeFree(handle) } catch (_: Exception) {}; handle = 0L } - } -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/NeonOps.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/NeonOps.kt deleted file mode 100644 index cfc66c9..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/NeonOps.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.kazeia.tts - -/** NEON SIMD optimized operations for TTS head argmax. */ -object NeonOps { - init { System.loadLibrary("neon_ops") } - - /** Argmax of hidden @ headWeights.T for one head. */ - external fun headArgmax(hidden: FloatArray, headWeights: FloatArray, vocab: Int, dim: Int): Int - - /** Batch argmax for all heads at once (avoids JNI overhead per head). */ - external fun headArgmaxBatch(hidden: FloatArray, allHeads: FloatArray, numHeads: Int, vocab: Int, dim: Int): IntArray - - /** Argmax with offset into a large weight buffer (avoids array copy). */ - external fun headArgmaxOffset(hidden: FloatArray, allHeads: FloatArray, offset: Int, vocab: Int, dim: Int): Int -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3BpeTokenizer.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3BpeTokenizer.kt deleted file mode 100644 index adc334a..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3BpeTokenizer.kt +++ /dev/null @@ -1,219 +0,0 @@ -package com.kazeia.tts - -import android.util.Log -import org.json.JSONObject -import java.io.File -import java.util.regex.Pattern - -/** - * Byte-level BPE tokenizer compatible with Qwen2/Qwen3 tokenizer. - * - * Why this file exists: - * The app needs to tokenize arbitrary LLM-generated French text on-device - * for the TTS pipeline (the reduced 1050-token table shipped previously - * couldn't handle free-form text). The reference is the HuggingFace - * Qwen2TokenizerFast — a GPT-2-style byte-level BPE. We reimplement it - * in Kotlin rather than linking libtokenizers.so so there's no new - * native dependency and the numbers are easy to audit. - * - * Algorithm (identical to GPT-2 / Qwen2): - * 1. Pre-tokenize the input with a regex that groups contractions, - * words, numbers, and whitespace into "word chunks" that BPE never - * merges across. - * 2. UTF-8 encode each chunk, then map each byte (0..255) to one of - * 256 printable Unicode code points via a fixed "byte encoder" table - * — this is the trick that lets a byte-level vocab fit inside JSON - * without invalid or control characters. - * 3. Apply BPE: repeatedly find the pair with the lowest rank in the - * merges list and merge it, until no more merges apply. Look up each - * resulting super-token in vocab.json to get the final token IDs. - * - * Bit-perfect compatibility notes: - * - The pre-tokenize regex below MUST match the one in tokenizer_config.json. - * Qwen2 uses the GPT-2 pattern with a couple of Unicode property - * extensions; Java regex supports these directly. - * - Byte encoder is canonical (see bytesToUnicode()). - * - Merges are rank-ordered: lower line number = higher priority, matching - * HuggingFace's `merges.txt` file ordering. - * - We do NOT add BOS/EOS or chat-template specials — the TTS prefill - * prepends its own 9-embed voice prefix that already handles role tokens. - */ -class Qwen3BpeTokenizer private constructor( - private val vocab: HashMap, - private val merges: HashMap, Int>, - private val byteEncoder: IntArray, -) { - companion object { - private const val TAG = "Qwen3BPE" - - // Qwen2/Qwen3 pre-tokenization regex. This is the exact pattern used - // by the HuggingFace Qwen2Tokenizer (adapted from GPT-2). Matches - // contractions ('s|'d|'ll|'ve|'re|'t), word chunks with a leading - // optional non-word char + letters, runs of digits, runs of - // punctuation, and whitespace runs with boundary semantics. - // - // Note on the character classes: Android's Pattern does NOT support - // UNICODE_CHARACTER_CLASS — so plain \\p{L} / \\p{N} would collapse - // to ASCII-only and break French accents ("é" → wrong token). Use - // \\p{IsAlphabetic} and \\p{IsDigit} instead; those ARE Unicode-aware - // out of the box (they map to ICU's IsAlphabetic / IsDigit properties - // in both JDK and Android runtimes). Output matches Python's Qwen2 - // tokenizer on French text. - private val PRE_TOKENIZE_PATTERN: Pattern = Pattern.compile( - "'s|'d|'ll|'ve|'re|'t" + - "|[^\\r\\n\\p{IsAlphabetic}\\p{IsDigit}]?\\p{IsAlphabetic}+" + - "|\\p{IsDigit}{1,3}" + - "| ?[^\\s\\p{IsAlphabetic}\\p{IsDigit}]+[\\r\\n]*" + - "|\\s*[\\r\\n]+" + - "|\\s+(?!\\S)" + - "|\\s+" - ) - - /** - * GPT-2 byte encoder: maps 0..255 → a printable Unicode codepoint. - * Ensures every possible byte has a visible, JSON-safe representation - * so a byte-level vocab can be stored as strings in vocab.json. - */ - private fun bytesToUnicode(): IntArray { - val bs = mutableListOf() - // Printable ASCII and common Latin blocks. - bs.addAll(('!'.code..'~'.code).toList()) - bs.addAll(('¡'.code..'¬'.code).toList()) - bs.addAll(('®'.code..'ÿ'.code).toList()) - val cs = bs.toMutableList() - // Every byte not in bs gets mapped to a code point past 255 so no - // existing character collides with it. - var n = 0 - val map = IntArray(256) - for (b in 0..255) { - if (b in bs) { - map[b] = b - } else { - map[b] = 256 + n - bs.add(b) // placeholder only, we've already recorded cs from the frozen snapshot - cs.add(256 + n) - n += 1 - } - } - return map - } - - fun load(modelDir: String): Qwen3BpeTokenizer { - val t0 = System.currentTimeMillis() - val vocabFile = File("$modelDir/vocab.json") - val mergesFile = File("$modelDir/merges.txt") - require(vocabFile.exists()) { "vocab.json missing at $modelDir" } - require(mergesFile.exists()) { "merges.txt missing at $modelDir" } - - val vocabJson = JSONObject(vocabFile.readText()) - val vocab = HashMap(vocabJson.length()) - val keys = vocabJson.keys() - while (keys.hasNext()) { - val k = keys.next() - vocab[k] = vocabJson.getInt(k) - } - - val merges = HashMap, Int>() - var rank = 0 - mergesFile.useLines { lines -> - for (line in lines) { - // Skip header / blanks. Qwen's merges.txt starts with - // "#version" which we simply filter out. - if (line.isBlank() || line.startsWith("#")) continue - val sp = line.indexOf(' ') - if (sp < 0) continue - merges[Pair(line.substring(0, sp), line.substring(sp + 1))] = rank - rank++ - } - } - - Log.i(TAG, "Loaded vocab=${vocab.size} merges=${merges.size} in ${System.currentTimeMillis()-t0}ms") - return Qwen3BpeTokenizer(vocab, merges, bytesToUnicode()) - } - } - - /** - * Convert a single pre-tokenized word (UTF-8 bytes encoded via the byte - * encoder into a string) into token IDs via BPE merges. Caches results so - * repeated common words (spaces, punctuation) only BPE once. - */ - private val bpeCache = HashMap() - - private fun bpeEncode(byteEncodedWord: String): IntArray { - bpeCache[byteEncodedWord]?.let { return it } - - // Start with one "sub-token" per Unicode code point (code points, - // not chars — surrogate pairs are handled automatically since the - // byte encoder only produces BMP codepoints by construction). - val parts = ArrayList(byteEncodedWord.length) - var i = 0 - while (i < byteEncodedWord.length) { - val cp = byteEncodedWord.codePointAt(i) - parts.add(String(Character.toChars(cp))) - i += Character.charCount(cp) - } - if (parts.size < 2) { - val id = vocab[parts.getOrElse(0) { "" }] ?: vocab[""] ?: 0 - val out = intArrayOf(id) - bpeCache[byteEncodedWord] = out - return out - } - - // Greedy lowest-rank merge, classic BPE. We scan for the pair with - // the smallest rank, merge ALL its occurrences, then repeat. This - // matches HF's reference implementation. - while (parts.size > 1) { - var bestRank = Int.MAX_VALUE - var bestIdx = -1 - for (k in 0 until parts.size - 1) { - val r = merges[Pair(parts[k], parts[k + 1])] ?: continue - if (r < bestRank) { bestRank = r; bestIdx = k } - } - if (bestIdx < 0) break - // Merge all non-overlapping occurrences of that exact pair. - val a = parts[bestIdx]; val b = parts[bestIdx + 1] - val merged = a + b - var k = 0 - val out = ArrayList(parts.size - 1) - while (k < parts.size) { - if (k < parts.size - 1 && parts[k] == a && parts[k + 1] == b) { - out.add(merged); k += 2 - } else { - out.add(parts[k]); k += 1 - } - } - parts.clear(); parts.addAll(out) - } - - val ids = IntArray(parts.size) - for (k in parts.indices) { - ids[k] = vocab[parts[k]] ?: vocab[""] ?: 0 - } - bpeCache[byteEncodedWord] = ids - return ids - } - - /** - * Encode text → token IDs using the full Qwen3 vocabulary. Does NOT - * prepend BOS/EOS — callers can add specials themselves. Unicode - * characters outside ASCII (e.g. French accents) are UTF-8 encoded and - * go through the byte encoder, so "é" and "ï" tokenize the same way as - * they do in Python. - */ - fun encode(text: String): IntArray { - val all = ArrayList(text.length / 2 + 4) - val matcher = PRE_TOKENIZE_PATTERN.matcher(text) - while (matcher.find()) { - val chunk = matcher.group() - // UTF-8 encode the chunk, then map each raw byte to its Unicode - // "byte-encoded" character. This produces the exact string that - // BPE merges operate on. - val bytes = chunk.toByteArray(Charsets.UTF_8) - val sb = StringBuilder(bytes.size) - for (b in bytes) sb.appendCodePoint(byteEncoder[b.toInt() and 0xff]) - val ids = bpeEncode(sb.toString()) - for (id in ids) all.add(id) - } - return all.toIntArray() - } -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt deleted file mode 100644 index 5afc735..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsEngine.kt +++ /dev/null @@ -1,5537 +0,0 @@ -package com.kazeia.tts - -import ai.onnxruntime.OnnxTensor -import ai.onnxruntime.OnnxJavaType -import ai.onnxruntime.OrtEnvironment -import ai.onnxruntime.OrtSession -import android.media.AudioAttributes -import android.media.AudioFormat -import android.media.AudioTrack -import android.util.Log -import com.kazeia.BuildConfig -import com.kazeia.core.TtsEngine -import com.kazeia.core.TtsResult -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlinx.coroutines.withContext -import java.io.File -import java.io.RandomAccessFile -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.nio.FloatBuffer -import java.nio.LongBuffer -import java.nio.IntBuffer -import kotlin.coroutines.resume - -/** - * Qwen3-TTS Engine — Full NPU pipeline for voice cloning TTS. - * - * Pipeline: - * 1. Talker NPU (ORT KV-cache decoder) → codec tokens (codebook 0) - * 2. Code Predictor NPU (QNN ONNX) → 16 codebooks - * 3. VQ decode (Kotlin) → quantized [1, 512, 60] - * 4. pre_conv NPU → preprocessor NPU → ConvNet decoder NPU → audio WAV - * - * All heavy computation on NPU. VQ decode is trivial CPU (codebook lookup). - */ -class Qwen3TtsEngine( - private val nativeLibDir: String, - private val context: android.content.Context? = null, - private val onLog: ((String) -> Unit)? = null -) : TtsEngine { - - companion object { - private const val TAG = "Qwen3TTS" - private const val SR = 24000 - private const val SEQ_LEN = 60 // Fixed NPU chunk size for decoder - private const val CHUNK_OVERLAP = 10 - private const val EFFECTIVE_CHUNK = SEQ_LEN - CHUNK_OVERLAP - private const val SAMPLES_PER_TOKEN = 1920 - private const val CODEC_OFFSET = 1050 - private const val NUM_CODEBOOKS = 16 - private const val CODEBOOK_SIZE = 2048 - private const val CODEBOOK_DIM = 256 - private const val HIDDEN_DIM = 512 - - // Talker KV-cache model constants - private const val TALKER_DIM = 1024 - private const val TALKER_VOCAB = 3072 // codec vocabulary size - private const val TALKER_LAYERS = 28 - private const val TALKER_HEADS = 8 - private const val TALKER_HEAD_DIM = 128 - private const val KV_LEN = 199 - private const val MAX_CONTEXT = 200 // KV_LEN + 1 - private const val MASK_NEG = -10000f - - // Code Predictor KV-cache constants - private const val CP_LAYERS = 5 - private const val CP_KV_HEADS = 8 - private const val CP_HEAD_DIM = 128 - private const val CP_KV_LEN = 16 // max 16 past positions (17 total with current) - - // Talker .pte constants - private const val TALKER_PTE_KV_LEN = 100 // must match .pte export (KV=64 caused quality loss) - - // Codec special token IDs (in talker's 3072 vocab space) - private const val CODEC_EOS = 2150 - private const val CODEC_BOS = 2149 - private const val CODEC_PAD = 2148 - private const val CODEC_THINK = 2154 - private const val CODEC_NOTHINK = 2155 - private const val CODEC_THINK_BOS = 2156 - private const val CODEC_THINK_EOS = 2157 - private const val CODEC_LANG_FR = 2061 - - // Chat template token IDs (reduced vocab) - private const val IM_START = 1048 - private const val IM_END = 1049 - private const val TOKEN_USER = 872 - private const val TOKEN_ASSISTANT = 1042 - private const val TOKEN_NEWLINE = 198 - - // Streaming decode: when true, BigVGAN dispatches a chunk's audio as - // soon as SEQ_LEN codes are ready from the talker/CP loop rather than - // waiting for all tokens. For long segments this overlaps the final - // BigVGAN passes with ongoing talker/CP work on Hexagon, cutting the - // first-audio latency by ~4 s. Short segments (? = null - private var damienVoiceSuffix: Array? = null - private var bpeTokenizer: Qwen3BpeTokenizer? = null - private var ttsBosEmbed: FloatArray? = null // [1024] - tts_bos text-side embedding - private var ttsEosEmbed: FloatArray? = null // [1024] - tts_eos text-side embedding - private var ttsPadEmbed: FloatArray? = null // [1024] - tts_pad text-side embedding - private var speakerEmbed: FloatArray? = null // [1024] - x-vector speaker embedding - - // VQ codebooks (loaded from numpy) - private var firstCodebook: FloatArray? = null // [2048, 256] - private var restCodebooks: Array? = null // 15 × [2048, 256] - private var firstOutputProj: FloatArray? = null // [512, 256] - private var restOutputProj: FloatArray? = null // [512, 256] - - // Code predictor embeddings [15, 2048, 1024] - private var cpEmbeddings: FloatArray? = null - - private var loaded = false - private var modelPath: String? = null - private var audioTrack: AudioTrack? = null - private var talkerUsesInt64Pos = false - private var talkerUsesCosSin = false - private var rotaryCos: FloatArray? = null - private var rotarySin: FloatArray? = null - private var cpUsesCosSin = false - private var cpRotaryCos: FloatArray? = null - private var cpRotarySin: FloatArray? = null - private var cpHeadsPath: String? = null // path dir for head_0..14.npy - private var cpHeadsCache: Array? = null // lazy-loaded heads cache (8MB each) - private var cpAllHeads: FloatArray? = null // all 15 heads concatenated [15*2048*1024] for batch NEON - private var cpPteModule: org.pytorch.executorch.Module? = null // ExecuTorch CP on NPU (JNI) - private var talkerPteModule: org.pytorch.executorch.Module? = null // ExecuTorch talker on NPU (JNI) - private var nativePipelineReady: Boolean = false // C++ native pipeline available - private var talkerPteRotaryCos: FloatArray? = null - private var talkerPteRotarySin: FloatArray? = null - private var useEtCp: Boolean = false // CP via ExecuTorch runner process (root) - private var cpEtSocket: java.net.Socket? = null - private var cpFixedKv: Boolean = false // GPU uses fixed-size KV (shift + mask) - // Décodeur Qwen3-TTS officiel exporté en .pte ExecuTorch CPU XNNPACK avec - // dynamic_shapes sur dim T (codes [1,16,T] → wav [1,1,T*1920]). Dimensions - // variables 1..512 tokens : décode toute la phrase d'un coup, pas de - // chunking ni padding ni heuristiques. - private var decoderPteModule: org.pytorch.executorch.Module? = null - - // Décodeur Qwen3-TTS pur ggml C++ — remplace decoderPteModule en priorité. - // GGUF en /data/local/tmp/kazeia/models/qwen3tts_decoder.gguf (325 MB, conv - // weights F16). RTF 1.05 sur OnePlus Pad 3 8 threads, validé bit-close. - // Voir kazeia-tts-decoder-ggml + memory project_tts_ggml_decoder_port. - private var decoderGgml: Qwen3TtsGgmlDecoder? = null - - // ggml-cpu (llama.cpp) in-process Talker + CP. Activated by - // /data/local/tmp/kazeia/force_cpu_talker_gguf when talker_f16.gguf, - // cp_f16.gguf, cp_heads.bin, cp_codec_embs.bin are all present. - // Goal: replace .pte fixed-KV path with dynamic-KV runtime so the - // "beg beg beg" chunk-transition artefact is eliminated structurally. - private var talkerCpuHandle: Long = 0 - private var cpCpuHandle: Long = 0 - private var useCpuPath: Boolean = false - - private var debugLogFile: java.io.File? = null - - /** - * Check a diagnostic flag file. Returns true only in DEBUG builds; release - * builds always return false so the file system check and any diagnostic - * code path is dead code (JIT-eliminated). Used for investigations that - * were valuable during development but must not leak into production: - * force_inject_pycb0, force_greedy_cb0, cb0_temp, force_python_codes, - * force_cpu_talker, force_cpu_talker_gguf. - * - * The cross-architecture tremor investigation (thesis-relevant) showed - * x86 Python ↔ ARM64 tablet is a numerical floor (~28% cb0 divergence), - * not fixable by runtime swap. These flags stay in tree for re-running - * the experiment on demand (demos, thesis figures), not for production. - */ - private fun diagFlag(name: String): Boolean = - BuildConfig.DEBUG && File("/data/local/tmp/kazeia/$name").exists() - - private fun diagFile(name: String): File? = - if (BuildConfig.DEBUG) File("/data/local/tmp/kazeia/$name").takeIf { it.exists() } else null - - /** - * Racine modèles résolue par KazeiaPaths : legacy `/data/local/tmp/kazeia/models` - * sur tablette dev, stockage externe app (rempli par l'installeur) sinon. - * Les chemins de DEBUG/dump/recherche restent en dur sur /data/local/tmp. - */ - private val modelsRoot: String get() = com.kazeia.KazeiaApplication.MODELS_DIR - - private fun nlog(msg: String) { - Log.i(TAG, msg) - Log.w(TAG, msg) - onLog?.invoke("[TTS] $msg") - try { - val f = debugLogFile ?: java.io.File("/data/local/tmp/kazeia/tts_debug.log") - f.appendText("${System.currentTimeMillis()} $msg\n") - } catch (_: Exception) { - // If /data/local/tmp fails, try app-internal dir - try { - val f2 = java.io.File("/data/local/tmp/kazeia/models/qwen3-tts-npu/tts_debug.log") - f2.appendText("${System.currentTimeMillis()} $msg\n") - debugLogFile = f2 - } catch (_: Exception) {} - } - } - - override suspend fun load(modelPath: String?, voiceId: String?) { - withContext(Dispatchers.IO) { - val path = modelPath ?: return@withContext - this@Qwen3TtsEngine.modelPath = path - try { - val t0 = System.currentTimeMillis() - ortEnv = OrtEnvironment.getEnvironment() - val htpPath = "$nativeLibDir/libQnnHtp.so" - - val qnnCacheDir = "$path/qnn_cache" - File(qnnCacheDir).mkdirs() - - fun loadQnn(name: String, fp16: Boolean = false): OrtSession { - val t = System.currentTimeMillis() - val opts = OrtSession.SessionOptions() - val qnnOpts = mutableMapOf( - "backend_path" to htpPath, - "qnn_context_cache_enable" to "1", - "qnn_context_cache_path" to "$qnnCacheDir/${name}.bin" - ) - if (fp16) { - qnnOpts["htp_graph_finalization_optimization_mode"] = "3" - qnnOpts["enable_htp_fp16_precision"] = "1" - } - opts.addQnn(qnnOpts) - val session = ortEnv!!.createSession("$path/$name/model.onnx", opts) - val mode = if (fp16) "QNN fp16" else "QNN" - nlog("$name ($mode): ${System.currentTimeMillis() - t}ms") - return session - } - - fun loadCpu(name: String, threads: Int = 8): OrtSession { - val t = System.currentTimeMillis() - val opts = OrtSession.SessionOptions() - opts.setIntraOpNumThreads(threads) - opts.setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT) - val session = ortEnv!!.createSession("$path/$name/model.onnx", opts) - nlog("$name (CPU ${threads}T): ${System.currentTimeMillis() - t}ms") - return session - } - - val gpuLib = "$nativeLibDir/libQnnGpu.so" - val hasGpu = File(gpuLib).exists() - - fun loadGpu(onnxPath: String, label: String): OrtSession { - val t = System.currentTimeMillis() - val opts = OrtSession.SessionOptions() - opts.addQnn(mapOf("backend_path" to gpuLib)) - val session = ortEnv!!.createSession(onnxPath, opts) - nlog("$label (GPU): ${System.currentTimeMillis() - t}ms") - return session - } - - // Speech decoder V2 on CPU. Two paths tried, both worse than CPU: - // - HTP: BigVGAN convolutions too slow to compile (timeout) - // - GPU Adreno via QNN GPU EP: model loads but per-phrase - // inference is ~3.5 s vs ~2 s on CPU (GPU/CPU memory transfer - // overhead dominates for this conv-heavy model) - // CPU 8-thread stays the practical optimum. - // Décodeur Qwen3-TTS pur ggml C++ — TENTÉ EN PREMIER (remplace .pte). - // GGUF avec conv weights F16, lib ARM optimisée (i8mm/dotprod/fp16/bf16). - // Validé bit-close vs PyTorch, RTF 1.05 sur OnePlus Pad 3 8 threads. - val decGgufFile = File("$modelsRoot/qwen3tts_decoder.gguf") - if (decGgufFile.exists()) { - try { - val tG = System.currentTimeMillis() - val ggml = Qwen3TtsGgmlDecoder(decGgufFile.absolutePath) - if (ggml.load()) { - decoderGgml = ggml - nlog("Décodeur ggml C++ loadé en ${System.currentTimeMillis() - tG}ms") - } else { - nlog("Décodeur ggml load failed → fallback .pte") - } - } catch (e: Throwable) { - nlog("Décodeur ggml exception: ${e.message} → fallback .pte") - } - } - - // Décodeur officiel Qwen3-TTS .pte ExecuTorch dyn-T : chargé UNIQUEMENT - // si le ggml C++ n'est pas dispo. Le .pte coûte 456 MB de RSS qu'on - // évite de payer en double quand le ggml fonctionne. - // Si le ggml échoue runtime, on lève l'exception (visible dans logs) - // plutôt que tomber sur V2 ONNX chunked legacy ("page beg beg"). - val decPte = File("$modelsRoot/qwen3tts_decoder_dynt.pte") - if (decoderGgml == null && decPte.exists()) { - try { - val tD = System.currentTimeMillis() - decoderPteModule = org.pytorch.executorch.Module.load( - decPte.absolutePath, - org.pytorch.executorch.Module.LOAD_MODE_FILE, - 1 - ) - val rc = decoderPteModule!!.loadMethod("forward") - if (rc != 0) { - nlog("Décodeur .pte loadMethod failed rc=$rc, on retombe sur ONNX V2") - decoderPteModule = null - } else { - nlog("Décodeur .pte (dyn-T) loadé en ${System.currentTimeMillis()-tD}ms") - } - } catch (e: Exception) { - nlog("Décodeur .pte load failed: ${e.message}") - decoderPteModule = null - } - } - - // Si décodeur ggml ou .pte chargé → marqueur CPU et skip ONNX V2 legacy - if (decoderGgml != null || decoderPteModule != null) { - decoderOnCpu = true - val active = if (decoderGgml != null) "ggml C++" else ".pte ExecuTorch" - nlog("Décodeur $active actif → on saute le chargement V2 ONNX") - } - // Décodeur dynamique-T : si présent dans /data/local/tmp/kazeia/decoder_dynt/, - // on charge ces ONNX (export avec dynamic_axes sur la dim T) qui acceptent - // n'importe quelle longueur de séquence — permet de décoder un segment entier - // d'un coup au lieu de chunker à SEQ_LEN=60 (élimine le « beg beg beg » des - // transitions de chunk BigVGAN). - // Skip V2 ONNX dyn-T loading aussi quand decoderGgml est présent - val dyntDir = if (decoderGgml != null || decoderPteModule != null) File("/dev/null") else File("/data/local/tmp/kazeia/decoder_dynt") - val v2Path = "$path/v2_pre_conv" - if (dyntDir.isDirectory && - File(dyntDir, "v2_pre_conv.onnx").exists() && - File(dyntDir, "v2_pre_transformer.onnx").exists() && - File(dyntDir, "v2_decoder_conv.onnx").exists()) { - nlog("Loading V2 dyn-T speech decoder (CPU ONNX)...") - fun loadCpuAbs(p: String, threads: Int): OrtSession { - val opts = OrtSession.SessionOptions() - opts.setIntraOpNumThreads(threads) - opts.setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT) - return ortEnv!!.createSession(p, opts) - } - preConv = loadCpuAbs("${dyntDir.absolutePath}/v2_pre_conv.onnx", 4) - preprocessor = loadCpuAbs("${dyntDir.absolutePath}/v2_pre_transformer.onnx", 4) - convDecoder = loadCpuAbs("${dyntDir.absolutePath}/v2_decoder_conv.onnx", 8) - decoderOnCpu = true - decoderDyntLayout = true - nlog("Speech decoder V2 dyn-T on CPU (no transpose, [1, 1024, T] layout)") - } else if (decoderGgml != null || decoderPteModule != null) { - val active = if (decoderGgml != null) "ggml C++" else ".pte" - nlog("Décodeur $active chargé : pas besoin du V2 ONNX legacy") - } else if (File("$v2Path/model.onnx").exists()) { - nlog("Loading V2 speech decoder (CPU ONNX)...") - preConv = loadCpu("v2_pre_conv", 4) - preprocessor = loadCpu("v2_pre_transformer", 4) - convDecoder = loadCpu("v2_decoder_conv", 8) // BigVGAN benefits from more threads - decoderOnCpu = true - nlog("Speech decoder V2 on CPU") - } else { - nlog("Loading speech decoder (QNN HTP)...") - preConv = loadQnn("pre_conv") - preprocessor = loadQnn("preprocessor") - convDecoder = loadQnn("conv_decoder") - nlog("Speech decoder on HTP") - } - - // Set ADSP library path for QNN HTP skel libs (needed by both Java and C++ paths) - android.system.Os.setenv("ADSP_LIBRARY_PATH", "$nativeLibDir;/data/local/tmp/kazeia/qnn_libs;/vendor/dsp/cdsp;/vendor/dsp", true) - - // Test overrides: skip backends to expose lower-priority talker paths. - // force_hexagon → skip .pte, use ggml-hexagon HMX - // force_cpu_talker → skip .pte AND Hexagon, use the CPU ONNX talker - // (used as an "EOS oracle" path because fp32 still - // produces natural codec_eos where fp16 NPU drifts.) - // force_hexagon is the production routing flag (not diagnostic): keep - // unconditional. force_cpu_talker is diagnostic (used to verify CPU-fp32 - // equivalence during the tremor investigation) → DEBUG only. - val forceHexagon = File("/data/local/tmp/kazeia/force_hexagon").exists() - val forceCpuTalker = diagFlag("force_cpu_talker") - // Voie validée 2026-05-01 (cf project_tts_pipeline_dynamique.md) : - // ggml-cpu Talker+CP + ggml C++ decoder. On force toujours cette voie, - // plus besoin du flag fichier `force_cpu_talker_gguf`. Évite le double - // chargement (cp_kv_v2 ONNX + cp_et_runner TCP) qui causait OOM en cohabitation - // avec la cascade LLM (RAM tablette ~16 GB plafonnée). - val forceCpuGguf = true - if (forceHexagon) nlog("force_hexagon flag detected: skipping .pte init") - if (forceCpuTalker) nlog("force_cpu_talker flag detected: skipping .pte AND hexagon init") - nlog("ggml-cpu Talker+CP path forced (validated 2026-05-01)") - - // ggml-cpu in-process Talker+CP via llama.cpp. Highest priority - // when force_cpu_talker_gguf is set and all assets are present. - // On success, sets useCpuPath=true and the rest of the .pte / - // Hexagon / ONNX init is skipped via early returns below. - if (forceCpuGguf) run { - // Préférer talker_f32 si présent : élimine le drift fp16 cumulatif - // qui faisait monter le rang EOS et provoquait le babillage du - // talker en ggml-cpu (cf. cp_f16→cp_f32 même classe de fix). - val tkrF32 = File("$modelsRoot/talker_f32.gguf") - val tkrModel = if (tkrF32.exists()) tkrF32 else File("$modelsRoot/talker_f16.gguf") - // Priorité : cp_q8_0 (84 MB, ~30 % plus rapide que f32, accumulator - // fp32 préservé) → cp_f32 (316 MB stable) → cp_f16 (overflow connu). - val cpQ8 = File("$modelsRoot/cp_q8_0.gguf") - val cpF32 = File("$modelsRoot/cp_f32.gguf") - val cpModel = when { - cpQ8.exists() -> cpQ8 - cpF32.exists() -> cpF32 - else -> File("$modelsRoot/cp_f16.gguf") - } - nlog("CP model selected : ${cpModel.name} (${cpModel.length() / 1024 / 1024} MB)") - val cpHeads = File("$modelsRoot/cp_heads.bin") - val cpEmbs = File("$modelsRoot/cp_codec_embs.bin") - if (!tkrModel.exists() || !cpModel.exists() || !cpHeads.exists() || !cpEmbs.exists()) { - nlog("force_cpu_talker_gguf: missing one of talker_f16.gguf/cp_f16.gguf/cp_heads.bin/cp_codec_embs.bin — skipping CPU path") - return@run - } - try { - val tT = System.currentTimeMillis() - talkerCpuHandle = TtsTalkerCpuJni.nativeInit(tkrModel.absolutePath) - nlog("Talker (ggml-cpu) init: ${System.currentTimeMillis() - tT}ms, handle=$talkerCpuHandle") - if (talkerCpuHandle == 0L) throw RuntimeException("talker nativeInit returned 0") - - val tC = System.currentTimeMillis() - cpCpuHandle = TtsCpCpuJni.nativeInit(cpModel.absolutePath, cpHeads.absolutePath, cpEmbs.absolutePath) - nlog("CP (ggml-cpu) init: ${System.currentTimeMillis() - tC}ms, handle=$cpCpuHandle") - if (cpCpuHandle == 0L) throw RuntimeException("cp nativeInit returned 0") - - useCpuPath = true - nlog("ggml-cpu Talker+CP path active — .pte / Hexagon / ONNX talker init will be skipped") - } catch (e: Exception) { - nlog("ggml-cpu init failed: ${e.message} — falling back to .pte/Hexagon/ONNX") - if (talkerCpuHandle != 0L) { try { TtsTalkerCpuJni.nativeFree(talkerCpuHandle) } catch (_: Exception) {}; talkerCpuHandle = 0L } - if (cpCpuHandle != 0L) { try { TtsCpCpuJni.nativeFree(cpCpuHandle) } catch (_: Exception) {}; cpCpuHandle = 0L } - useCpuPath = false - } - } - - // Load .pte modules via Java JNI (native pipeline uses same instances). - // CP .pte can load regardless of forceHexagon/forceCpuTalker — it only touches - // the NPU for its own graph. The talker .pte is separately gated below. - // Running CP on the NPU alongside a Hexagon talker MAY trigger DSP contention - // (memory note: "ggml-hexagon and ExecuTorch QNN cannot share CDSP"). This - // is exactly what we're testing. - run { - val etModel = File("/data/local/tmp/kazeia/models/cp_transformer_fp16.pte") - if (!useCpuPath && !forceCpuTalker && etModel.exists() && cpPteModule == null) { - try { - val t0 = System.currentTimeMillis() - cpPteModule = org.pytorch.executorch.Module.load( - etModel.absolutePath, - org.pytorch.executorch.Module.LOAD_MODE_FILE, - 1 - ) - nlog("CP .pte JNI loaded: ${System.currentTimeMillis() - t0}ms") - val t1 = System.currentTimeMillis() - val lmResult = cpPteModule!!.loadMethod("forward") - nlog("CP .pte loadMethod: ${System.currentTimeMillis() - t1}ms, result=$lmResult") - if (lmResult != 0) { - nlog("CP .pte loadMethod failed ($lmResult), disabling JNI") - cpPteModule = null - } else { - // Register CP module for native pipeline - cpPteModule!!.nativeSetCpModule() - nlog("CP module registered for native pipeline") - } - } catch (e: Exception) { - nlog("CP .pte JNI failed: ${e.message}") - cpPteModule = null - } - } - // Load talker .pte JNI (same QNN context, no DSP contention with CP .pte) - val talkerPte = File("/data/local/tmp/kazeia/models/talker_transformer_fp16.pte") - if (!useCpuPath && !forceHexagon && !forceCpuTalker && talkerPte.exists() && cpPteModule != null && talkerPteModule == null) { - try { - val t0 = System.currentTimeMillis() - talkerPteModule = org.pytorch.executorch.Module.load( - talkerPte.absolutePath, - org.pytorch.executorch.Module.LOAD_MODE_FILE, - 1 - ) - val lm = talkerPteModule!!.loadMethod("forward") - nlog("Talker .pte JNI loaded+compiled: ${System.currentTimeMillis() - t0}ms, result=$lm") - if (lm != 0) { nlog("Talker .pte loadMethod failed"); talkerPteModule = null } - else { - val path = "/data/local/tmp/kazeia/models" - talkerPteRotaryCos = loadNpy("$path/talker_pte_rotary_cos.npy") - talkerPteRotarySin = loadNpy("$path/talker_pte_rotary_sin.npy") - nlog("Talker .pte rotary: ${talkerPteRotaryCos?.size} floats") - - // Warmup both models: first forward() triggers QNN DSP compilation (~7s) - // Better to pay this cost at init than at first pipeline run - val tw = System.currentTimeMillis() - try { - val dE = FloatArray(TALKER_DIM) - val dM = FloatArray(TALKER_PTE_KV_LEN) { -1e9f }; dM[TALKER_PTE_KV_LEN - 1] = 0f - val dC = FloatArray(TALKER_HEAD_DIM) { 1f } - val dS = FloatArray(TALKER_HEAD_DIM) - val tkvSz = TALKER_HEADS * TALKER_PTE_KV_LEN * TALKER_HEAD_DIM - val ins = mutableListOf( - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(dE, longArrayOf(1,1,TALKER_DIM.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(dM, longArrayOf(1,1,1,TALKER_PTE_KV_LEN.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(dC, longArrayOf(1,1,TALKER_HEAD_DIM.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(dS, longArrayOf(1,1,TALKER_HEAD_DIM.toLong()))) - ) - for (i in 0 until TALKER_LAYERS * 2) ins.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(FloatArray(tkvSz), longArrayOf(1,TALKER_HEADS.toLong(),TALKER_PTE_KV_LEN.toLong(),TALKER_HEAD_DIM.toLong())))) - talkerPteModule!!.forward(*ins.toTypedArray()) - nlog("Talker warmup: ${System.currentTimeMillis() - tw}ms") - } catch (e: Exception) { nlog("Talker warmup failed: ${e.message}") } - - // CP warmup - val cw = System.currentTimeMillis() - try { - val ckvSz = CP_KV_HEADS * CP_KV_LEN * CP_HEAD_DIM - val cIns = mutableListOf( - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(FloatArray(TALKER_DIM), longArrayOf(1,1,TALKER_DIM.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(FloatArray(CP_KV_LEN){-1e9f}.also{it[CP_KV_LEN-1]=0f}, longArrayOf(1,1,1,CP_KV_LEN.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(FloatArray(CP_HEAD_DIM){1f}, longArrayOf(1,1,CP_HEAD_DIM.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(FloatArray(CP_HEAD_DIM), longArrayOf(1,1,CP_HEAD_DIM.toLong()))) - ) - for (i in 0 until CP_LAYERS * 2) cIns.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(FloatArray(ckvSz), longArrayOf(1,CP_KV_HEADS.toLong(),CP_KV_LEN.toLong(),CP_HEAD_DIM.toLong())))) - cpPteModule!!.forward(*cIns.toTypedArray()) - nlog("CP warmup: ${System.currentTimeMillis() - cw}ms") - } catch (e: Exception) { nlog("CP warmup failed: ${e.message}") } - - // Native pipeline already initialized above - } - } catch (e: Exception) { - nlog("Talker .pte JNI failed: ${e.message}") - talkerPteModule = null - } - } - } - - // Talker: skip Hexagon if .pte talker+CP are both loaded (avoids DSP contention) - val talkerT = System.currentTimeMillis() - val hexRunner = File("/data/local/tmp/kazeia/llama-hex/llama-tts-talker") - val hexModel = File("/data/local/tmp/kazeia/models/talker_f16.gguf") - val cpuOnnx = File("$path/talker_kv_cpu/model.onnx") - if (useCpuPath) { - nlog("Talker+CP using ggml-cpu (llama.cpp) — skipping .pte/Hexagon/ONNX talker init") - } else if (talkerPteModule != null && cpPteModule != null) { - nlog("Talker+CP using .pte JNI NPU — skipping Hexagon runners") - } else if (forceCpuTalker) { - nlog("force_cpu_talker: skipping Hexagon, will use CPU ONNX") - } else { - if (hexRunner.exists() && hexModel.exists()) { - if (hexStartRunner()) { - useHexagonTalker = true - nlog("Talker using Hexagon NPU (HMX FP16)") - } else { - nlog("Hexagon talker runner failed, using CPU") - } - } - val hexCpRunner = File("/data/local/tmp/kazeia/llama-hex/llama-tts-cp") - val hexCpModel = File("/data/local/tmp/kazeia/models/cp_f16.gguf") - if (hexCpRunner.exists() && hexCpModel.exists()) { - if (hexStartCpRunner()) { - useHexagonCp = true - nlog("CP using CPU GGUF (avoids Hexagon HMX NaN bug)") - } else { - nlog("CP CPU runner failed, falling back to ONNX") - } - } - } - // Fallback: CPU ONNX for talker if hexagon failed (skipped when ggml-cpu is active) - if (!useCpuPath && !useHexagonTalker) { - // Try new M-RoPE ONNX: GPU Adreno first (fast fp16), fallback CPU - val mropeOnnx = File("$path/talker_kv_cpu/model.onnx") - if (mropeOnnx.exists() && mropeOnnx.length() > 1_000_000) { - val talkerOpts = OrtSession.SessionOptions() - val gpuLib = "$nativeLibDir/libQnnGpu.so" - if (File(gpuLib).exists()) { - talkerOpts.addQnn(mapOf("backend_path" to gpuLib)) - nlog("Talker ONNX: loading on GPU Adreno...") - } else { - talkerOpts.setIntraOpNumThreads(6) - talkerOpts.setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT) - nlog("Talker ONNX: loading on CPU 6T...") - } - talkerKv = ortEnv!!.createSession(mropeOnnx.absolutePath, talkerOpts) - talkerUsesCosSin = true - // Load rotary tables - val cosFile = File("$path/talker_kv_cpu/talker_rotary_cos.npy") - val sinFile = File("$path/talker_kv_cpu/talker_rotary_sin.npy") - if (cosFile.exists()) rotaryCos = loadNpy(cosFile.absolutePath) - if (sinFile.exists()) rotarySin = loadNpy(sinFile.absolutePath) - nlog("talker_kv M-RoPE (CPU 6T): ${System.currentTimeMillis() - talkerT}ms, cos/sin=${rotaryCos?.size}") - } else if (cpuOnnx.exists()) { - val talkerOpts = OrtSession.SessionOptions() - talkerOpts.setIntraOpNumThreads(6) - talkerOpts.setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT) - talkerKv = ortEnv!!.createSession(cpuOnnx.absolutePath, talkerOpts) - talkerUsesInt64Pos = true - nlog("talker_kv legacy (CPU fp32 6T): ${System.currentTimeMillis() - talkerT}ms") - } else { - nlog("WARNING: No talker model available (no hexagon, no CPU ONNX)") - } - } - // CP V2 ONNX + cp_et_runner TCP : DESACTIVES quand ggml-cpu CP actif. - // Quand `useCpuPath` est true, le pipeline va via TtsCpCpuJni (cpCpuHandle) - // — charger en plus l'ONNX V2 (~300-500 MB) + spawner cp_et_runner TCP - // (process séparé) ne sert à rien et explosait le budget RAM en cohabitation - // avec la cascade LLM (OOM crash 2026-05-02). - if (!useCpuPath) { - run { - val cpV2 = File("$path/cp_kv_v2/model.onnx") - if (cpV2.exists() && cpKv == null) { - try { - if (!useHexagonCp && (cpPteModule == null || (useHexagonTalker && talkerPteModule == null))) { - val etModel = File("/data/local/tmp/kazeia/models/cp_transformer_fp16.pte") - val etRunner = File("/data/local/tmp/kazeia/cp_et_runner") - if (etRunner.exists() && etModel.exists()) { - if (hexStartCpEtRunner()) { - useEtCp = true - nlog("CP using ExecuTorch NPU runner (TCP, root)") - } else { - nlog("CP ET runner failed to start") - } - } - } - cpKv = loadCpu("cp_kv_v2") - cpUsesCosSin = true - cpRotaryCos = loadNpy("$path/cp_kv_v2/cp_rotary_cos.npy") - cpRotarySin = loadNpy("$path/cp_kv_v2/cp_rotary_sin.npy") - cpHeadsPath = "$path/cp_kv_v2/cp_heads.npy" - nlog("CP V2 ONNX loaded, cos/sin=${cpRotaryCos?.size}") - } catch (e: Exception) { - nlog("CP V2 ONNX failed: ${e.message}") - } - } - } - } else { - nlog("CP V2 ONNX skip (ggml-cpu CP actif)") - } - - // Load dual embedding tables for talker - textEmbeds = loadNpy("$path/text_embeds_projected.npy") - nlog("Text embeddings: ${textEmbeds!!.size / TALKER_DIM} × $TALKER_DIM") - - // Stage 2 assets: full-vocab text embeddings + voice prefix + BPE. - // All three are optional — if any is missing we fall back to the - // legacy 1050-token path. This keeps the app bootable during - // asset rollout and avoids turning a missing file into a crash. - try { - val fullEmbFile = File("$path/text_embeds_full_fp16.bin") - val prefixFile = File("$path/damien_voice_prefix.bin") - val tokDir = File("$path/qwen3_tokenizer") - if (fullEmbFile.exists() && prefixFile.exists() && tokDir.isDirectory) { - val tE0 = System.currentTimeMillis() - // Memory-map the fp16 embeddings table instead of heap- - // loading it. Without mmap, the 296 MB ByteArray plus - // the 125 MB cp_embeddings FloatArray loaded a few - // lines below overrun the ~536 MB large-heap limit and - // the app OOMs during init. mmap pages the file via - // the kernel and keeps zero bytes on the Java heap. - val expectedBytes = textEmbedsFullLen.toLong() * TALKER_DIM * 2 - if (fullEmbFile.length() != expectedBytes) { - nlog("text_embeds_full_fp16 size mismatch (got ${fullEmbFile.length()}, expected $expectedBytes) — disabling on-device text") - } else { - textEmbedsFullChan = java.io.RandomAccessFile(fullEmbFile, "r").channel - textEmbedsFullBuf = textEmbedsFullChan!!.map( - java.nio.channels.FileChannel.MapMode.READ_ONLY, 0L, expectedBytes - ).order(ByteOrder.LITTLE_ENDIAN) - nlog("Full-vocab text embeddings mmap: ${textEmbedsFullLen} × $TALKER_DIM fp16 (${expectedBytes/1024/1024}MB, off-heap) in ${System.currentTimeMillis()-tE0}ms") - } - - val pb = ByteBuffer.wrap(prefixFile.readBytes()).order(ByteOrder.LITTLE_ENDIAN) - val nPref = pb.int; val dimPref = pb.int - if (nPref == 9 && dimPref == TALKER_DIM) { - damienVoicePrefix = Array(9) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = pb.float } } - nlog("Damien voice prefix: $nPref × $dimPref") - } else { - nlog("damien_voice_prefix.bin header mismatch ($nPref × $dimPref) — disabling on-device text") - } - - // Voice SUFFIX — the 2 fixed positions that close out - // the prefill after text tokens. Empirically invariant - // across segments of the same speaker (diff = 0.0). - val suffixFile = File("$path/damien_voice_suffix.bin") - if (suffixFile.exists()) { - val sb = ByteBuffer.wrap(suffixFile.readBytes()).order(ByteOrder.LITTLE_ENDIAN) - val nSuf = sb.int; val dimSuf = sb.int - if (nSuf == 2 && dimSuf == TALKER_DIM) { - damienVoiceSuffix = Array(2) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = sb.float } } - nlog("Damien voice suffix: $nSuf × $dimSuf") - } - } else { - nlog("damien_voice_suffix.bin missing — on-device text will lack closure markers") - } - - if (textEmbedsFullBuf != null && damienVoicePrefix != null && damienVoiceSuffix != null) { - bpeTokenizer = Qwen3BpeTokenizer.load(tokDir.absolutePath) - nlog("Stage 2 on-device text path ready (BPE + full embeds + voice prefix)") - } - } else { - nlog("Stage 2 assets not fully present (full=$fullEmbFile, prefix=$prefixFile, tok=$tokDir) — legacy path only") - } - } catch (e: Exception) { - nlog("Stage 2 asset load failed: ${e.message} — legacy path only") - textEmbedsFullBuf = null; damienVoicePrefix = null; bpeTokenizer = null - try { textEmbedsFullChan?.close() } catch (_: Exception) {} - textEmbedsFullChan = null - } - - codecEmbedding = loadNpy("$path/codec_embedding.npy") - nlog("Codec embedding: ${codecEmbedding!!.size / TALKER_DIM} × $TALKER_DIM") - val ttsSpecial = loadNpy("$path/tts_special_embeds.npy") // [3, 1024] = bos, eos, pad - ttsBosEmbed = ttsSpecial.sliceArray(0 until TALKER_DIM) - ttsEosEmbed = ttsSpecial.sliceArray(TALKER_DIM until 2 * TALKER_DIM) - ttsPadEmbed = ttsSpecial.sliceArray(2 * TALKER_DIM until 3 * TALKER_DIM) - nlog("TTS special embeddings loaded") - - // Load speaker embedding (x-vector for voice cloning) - val spkFile = File("$path/speaker_embedding.npy") - if (spkFile.exists()) { - speakerEmbed = loadNpy(spkFile.absolutePath) - nlog("Speaker embedding: ${speakerEmbed!!.size} floats, norm=${Math.sqrt(speakerEmbed!!.sumOf { (it * it).toDouble() })}") - } - - // Load VQ codebooks - loadVqCodebooks(path) - - // Load code predictor embeddings - cpEmbeddings = loadNpy("$path/code_predictor_embeddings.npy") - nlog("CP embeddings: ${cpEmbeddings!!.size} floats") - - loaded = true - nlog("Qwen3-TTS loaded in ${System.currentTimeMillis() - t0}ms") - } catch (e: Exception) { - nlog("ERROR: ${e.message}") - e.printStackTrace() - } - } - } - - override fun isLoaded(): Boolean = loaded - - /** - * Hot-swap the speaker prefix/suffix embeddings used for voice - * conditioning. [voicePath] is a WAV path like - * `/…/voix/elodie.wav` — we derive the voice id from its basename - * and look for matching `_voice_prefix.bin` + `_voice_suffix.bin` - * in the model dir. If both files exist they replace the current - * [damienVoicePrefix] / [damienVoiceSuffix] arrays so the next - * segment generated uses the new voice. If either file is missing - * we log a warning and keep the current voice — per-voice - * prefix/suffix files are offline-generated via - * scripts/prepare_tts_native.py; run once per voice WAV and - * `adb push` into the model dir to enable. - * - * Thread-safety: the arrays are read by the synth worker on - * Dispatchers.IO; replacing a reference via a volatile var is - * atomic on the JVM so a mid-segment replacement just takes - * effect on the next segment boundary. - */ - fun setVoice(voicePath: String) { - val modelDir = "$modelsRoot/qwen3-tts-npu" - val id = java.io.File(voicePath).nameWithoutExtension.lowercase() - val prefixFile = java.io.File("$modelDir/${id}_voice_prefix.bin") - val suffixFile = java.io.File("$modelDir/${id}_voice_suffix.bin") - if (!prefixFile.exists() || !suffixFile.exists()) { - nlog("Voice '$id' not available (missing ${prefixFile.name} or ${suffixFile.name}); keeping current voice. " + - "Run scripts/prepare_tts_native.py with this WAV to generate the files.") - return - } - try { - val pBytes = prefixFile.readBytes() - val pHead = java.nio.ByteBuffer.wrap(pBytes).order(java.nio.ByteOrder.LITTLE_ENDIAN) - val nPref = pHead.int; val dimPref = pHead.int - if (dimPref != TALKER_DIM) throw IllegalStateException("prefix dim $dimPref != $TALKER_DIM") - val newPrefix = Array(nPref) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = pHead.float } } - - val sBytes = suffixFile.readBytes() - val sHead = java.nio.ByteBuffer.wrap(sBytes).order(java.nio.ByteOrder.LITTLE_ENDIAN) - val nSuf = sHead.int; val dimSuf = sHead.int - if (dimSuf != TALKER_DIM) throw IllegalStateException("suffix dim $dimSuf != $TALKER_DIM") - val newSuffix = Array(nSuf) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = sHead.float } } - - damienVoicePrefix = newPrefix - damienVoiceSuffix = newSuffix - nlog("Voice switched to '$id' ($nPref prefix + $nSuf suffix embeds)") - } catch (e: Exception) { - nlog("Voice swap failed for '$id': ${e.message}") - } - } - - override suspend fun synthesize(text: String, language: String): TtsResult { - return withContext(Dispatchers.IO) { - val audio = generateSpeech(text, language) - TtsResult(audioData = audio, sampleRate = SR, durationMs = audio.size * 1000L / SR) - } - } - - override suspend fun synthesizeAndPlay( - text: String, language: String, - onStart: (() -> Unit)?, onComplete: (() -> Unit)? - ) { - withContext(Dispatchers.IO) { - val codebooks = generateCodebooks(text, language) - if (codebooks == null) { onComplete?.invoke(); return@withContext } - - val (allCodebooks, numRealTokens) = codebooks - - // Release DSP for QNN decoder ONLY if decoder is on HTP (not GPU) - if (!decoderOnGpu && (useHexagonTalker || useHexagonCp)) { - hexStopRunner() - } - - onStart?.invoke() - - // Decode and play in streaming chunks - val track = AudioTrack.Builder() - .setAudioAttributes(AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_MEDIA) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build()) - .setAudioFormat(AudioFormat.Builder() - .setEncoding(AudioFormat.ENCODING_PCM_16BIT) - .setSampleRate(SR) - .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) - .build()) - .setBufferSizeInBytes(SAMPLES_PER_TOKEN * 20 * 2) // ~1.6s buffer - .setTransferMode(AudioTrack.MODE_STREAM) - .build() - track.play() - audioTrack = track - - val t3 = System.currentTimeMillis() - var pos = 0 - while (pos < numRealTokens) { - val chunkEnd = minOf(pos + EFFECTIVE_CHUNK, numRealTokens) - val chunkTokens = chunkEnd - pos - - // Build chunk codebooks padded to SEQ_LEN - val chunkCodes = Array(NUM_CODEBOOKS) { cb -> - IntArray(SEQ_LEN) { t -> - val srcIdx = pos + t - if (srcIdx < numRealTokens) allCodebooks[cb][srcIdx] else 0 - } - } - - val quantized = vqDecode(chunkCodes) - val chunkAudio = runSpeechDecoder(quantized) - - // Trim and crossfade - if (pos == 0) { - val keepSamples = minOf(chunkTokens * SAMPLES_PER_TOKEN, chunkAudio.size) - track.write(chunkAudio, 0, keepSamples) - } else { - val skipSamples = CHUNK_OVERLAP * SAMPLES_PER_TOKEN - val keepSamples = minOf(chunkTokens * SAMPLES_PER_TOKEN, chunkAudio.size - skipSamples) - if (keepSamples > 0 && skipSamples < chunkAudio.size) { - track.write(chunkAudio, skipSamples, keepSamples) - } - } - - pos += EFFECTIVE_CHUNK - } - nlog("Streaming decode: ${System.currentTimeMillis() - t3}ms") - - track.stop() - track.release() - audioTrack = null - onComplete?.invoke() - } - } - - override fun stop() { - audioTrack?.apply { - try { stop() } catch (_: Exception) {} - release() - } - audioTrack = null - } - - /** Generate codebooks only (no decode). Returns (allCodebooks[16][padLen], numRealTokens). */ - private fun generateCodebooks(text: String, language: String): Pair, Int>? { - if (!loaded) return null - val t0 = System.currentTimeMillis() - nlog("Generating codebooks: '$text' [$language]") - try { - // Stage 2 on-device path : BPE + full vocab mmap embeddings. - // On NE charge PLUS phrase_embeds.bin — c'était un cache pré-calculé pour - // UNE phrase fixe de test, qui shadowait le texte réel et provoquait la - // dégénération du Talker (cache embeds vs texte demandé décorrélés). - // textEmbFromFull() lit le mmap 151936 × 1024 fp16 — compatible BPE IDs. - val tokenIds = tokenizeText(text) - val textEmbedsList: List = if (textEmbedsFullBuf != null) { - tokenIds.map { textEmbFromFull(it) } - } else { - tokenIds.map { textEmb(it) } - } - nlog("Tokenized ${tokenIds.size} BPE tokens (Stage 2 path, fullEmb=${textEmbedsFullBuf != null})") - val maxGen = MAX_CONTEXT - 15 - val allCodesArray = runInterleavedGeneration(textEmbedsList, maxGen) - val genMs = System.currentTimeMillis() - t0 - nlog("Interleaved gen: ${genMs}ms, ${allCodesArray.size} tokens") - if (allCodesArray.isEmpty()) return null - - val numRealTokens = allCodesArray.size - val padLen = maxOf(numRealTokens, SEQ_LEN) - val allCodebooks = Array(NUM_CODEBOOKS) { cb -> - IntArray(padLen) { t -> if (t < numRealTokens) allCodesArray[t][cb] else 0 } - } - return Pair(allCodebooks, numRealTokens) - } catch (e: Exception) { - nlog("ERROR: ${e.message}") - return null - } - } - - private fun generateSpeech(text: String, language: String): ShortArray { - if (!loaded) return ShortArray(0) - val t0 = System.currentTimeMillis() - nlog("Generating: '$text' [$language]") - - try { - // Step 1: Stage 2 on-device path. NE PAS lire phrase_embeds.bin (cache - // figé d'une phrase de test, cf. dégénération Talker). - val tokenIds = tokenizeText(text) - val textEmbedsList: List = if (textEmbedsFullBuf != null) { - tokenIds.map { textEmbFromFull(it) } - } else { - tokenIds.map { textEmb(it) } - } - nlog("Text tokens: ${tokenIds.size}, fullEmb=${textEmbedsFullBuf != null}") - - // Step 2: Interleaved talker + code predictor → all 16 codebooks - val t1 = System.currentTimeMillis() - // Hard safety limit; actual stop is controlled by post-text step counter below. - val maxGen = MAX_CONTEXT - 15 - val allCodesArray = runInterleavedGeneration(textEmbedsList, maxGen) - nlog("Interleaved gen: ${System.currentTimeMillis() - t1}ms, ${allCodesArray.size} tokens") - if (allCodesArray.isEmpty()) return ShortArray(0) - - val numRealTokens = allCodesArray.size - - // Reshape to [16][totalTokens] for decoder (pad to SEQ_LEN if needed) - val padLen = maxOf(numRealTokens, SEQ_LEN) - val allCodebooks = Array(NUM_CODEBOOKS) { cb -> - IntArray(padLen) { t -> if (t < numRealTokens) allCodesArray[t][cb] else 0 } - } - - // Release DSP if decoder needs HTP (not GPU or CPU) - if (!decoderOnCpu && !decoderOnGpu && (useHexagonTalker || useHexagonCp)) { - hexStopRunner() - } - - // Step 3: VQ → decoder → audio (chunked) - val t3 = System.currentTimeMillis() - val audio = decodeChunked(allCodebooks, numRealTokens) - nlog("Decode (chunked): ${System.currentTimeMillis() - t3}ms") - - val totalMs = System.currentTimeMillis() - t0 - val audioDur = audio.size.toFloat() / SR - nlog("Total: ${totalMs}ms for ${audioDur}s audio (RTF ${totalMs / 1000f / audioDur})") - - return audio - } catch (e: Exception) { - nlog("ERROR: ${e.message}") - e.printStackTrace() - return ShortArray(0) - } - } - - // ==================== Tokenization ==================== - - /** - * Simple text tokenization. Maps known words to reduced vocab IDs. - * For production, this should be replaced with proper BPE tokenization. - */ - private fun tokenizeText(text: String): IntArray { - // Stage 2 path : BPE Qwen3 byte-level — gère arbitraire FR (accents, ponctuation, - // contractions). Le dico stub plus bas est legacy (tokens réduits 1050) et ne couvre - // qu'une dizaine de mots ; à n'utiliser que si le BPE n'est pas chargé. - bpeTokenizer?.let { bpe -> - val ids = bpe.encode(text) - nlog("BPE tokenized '${text.take(60)}…' → ${ids.size} tokens") - return ids - } - nlog("WARN: BPE tokenizer not loaded, fallback dictionnaire stub (couverture ~10 mots)") - // Known word → reduced vocab ID mappings (from Qwen3 tokenizer + token_mapping) - val knownTokens = mapOf( - "Bonjour" to 1043, - "bonjour" to 1028, - " bonjour" to intArrayOf(220, 1028), // space + bonjour - "Oui" to 1006, - "Non" to 966, - "Merci" to 1035, - "salut" to 1023, - "." to 13, - "," to 11, - "!" to 0, - "?" to 30, - " " to 220, - ) - - // Try exact match first - val exactId = knownTokens[text] - if (exactId is Int) return intArrayOf(exactId) - - // Try word-by-word tokenization - val tokens = mutableListOf() - var remaining = text - while (remaining.isNotEmpty()) { - var matched = false - // Try longest match - for (len in minOf(remaining.length, 20) downTo 1) { - val candidate = remaining.substring(0, len) - val id = knownTokens[candidate] - if (id != null) { - when (id) { - is Int -> tokens.add(id) - is IntArray -> tokens.addAll(id.toList()) - } - remaining = remaining.substring(len) - matched = true - break - } - } - if (!matched) { - // Skip unknown character - nlog("WARN: Unknown token for '${remaining.first()}'") - remaining = remaining.substring(1) - } - } - - if (tokens.isEmpty()) { - nlog("WARN: No tokens from text '$text', using Bonjour fallback") - return intArrayOf(1043) // "Bonjour" - } - return tokens.toIntArray() - } - - // ==================== Talker KV-Cache Generation ==================== - - // ==================== Embedding Helpers ==================== - - private fun textEmb(reducedId: Int): FloatArray { - val e = FloatArray(TALKER_DIM) - System.arraycopy(textEmbeds!!, reducedId.coerceIn(0, 1049) * TALKER_DIM, e, 0, TALKER_DIM) - return e - } - - private fun codecEmb(codecIdx: Int): FloatArray { - val e = FloatArray(TALKER_DIM) - System.arraycopy(codecEmbedding!!, codecIdx.coerceIn(0, TALKER_VOCAB - 1) * TALKER_DIM, e, 0, TALKER_DIM) - return e - } - - private fun cpEmb(codebookIdx: Int, tokenIdx: Int): FloatArray { - // cpEmbeddings is [15, 2048, 1024] flattened - val e = FloatArray(TALKER_DIM) - val offset = (codebookIdx * CODEBOOK_SIZE + tokenIdx.coerceIn(0, CODEBOOK_SIZE - 1)) * TALKER_DIM - System.arraycopy(cpEmbeddings!!, offset, e, 0, TALKER_DIM) - return e - } - - private fun sumEmb(a: FloatArray, b: FloatArray): FloatArray { - val r = FloatArray(TALKER_DIM) - for (i in 0 until TALKER_DIM) r[i] = a[i] + b[i] - return r - } - - private fun addEmb(dst: FloatArray, src: FloatArray) { - for (i in 0 until TALKER_DIM) dst[i] += src[i] - } - - /** - * Build prefill embeddings with speaker embedding (voice cloning). - * textEmbedsList: pre-computed text embeddings (one per text token) - */ - private fun buildPrefillEmbeddings(textEmbedsList: List): List { - val padE = ttsPadEmbed ?: return emptyList() - val bosE = ttsBosEmbed ?: return emptyList() - val spkE = speakerEmbed - - val embeddings = mutableListOf() - - // 1. Role: <|im_start|>assistant\n - embeddings.add(textEmb(IM_START)) - embeddings.add(textEmb(TOKEN_ASSISTANT)) - embeddings.add(textEmb(TOKEN_NEWLINE)) - - // 2. Codec control: [think, think_bos, lang_fr, think_eos] + tts_pad - for (cc in intArrayOf(CODEC_THINK, CODEC_THINK_BOS, CODEC_LANG_FR, CODEC_THINK_EOS)) { - embeddings.add(sumEmb(padE, codecEmb(cc))) - } - - // 3. Speaker embedding (x-vector) - if (spkE != null) { - embeddings.add(sumEmb(padE, spkE)) - } - - // 4. codec_pad + tts_bos - embeddings.add(sumEmb(bosE, codecEmb(CODEC_PAD))) - - // 5. First text token + codec_bos - if (textEmbedsList.isNotEmpty()) { - embeddings.add(sumEmb(textEmbedsList[0], codecEmb(CODEC_BOS))) - } - - nlog("Prefill: ${embeddings.size} tokens (3 role + 4 ctrl + ${if (spkE != null) "1 spk + " else ""}1 bos + 1 text)") - return embeddings - } - - // ==================== Hexagon NPU Talker ==================== - - private val HEX_DIR = "/data/local/tmp/kazeia/llama-hex" - private val HEX_INPUT = "/data/local/tmp/kazeia/tts_input.bin" - private val HEX_OUTPUT = "/data/local/tmp/kazeia/tts_logits.bin" - private val HEX_CONTROL = "/data/local/tmp/kazeia/tts_control.txt" - private val TALKER_SOCK = "/data/local/tmp/kazeia/talker.sock" - private val CP_SOCK = "/data/local/tmp/kazeia/cp.sock" - private val CP_ET_SOCK = "/data/local/tmp/kazeia/cp_et.sock" - private var talkerSocket: android.net.LocalSocket? = null - private var cpSocket: android.net.LocalSocket? = null - private var useHexagonCp = false - - /** Run a command with su. Returns false if su is not available. */ - private fun suExec(cmd: String): Boolean { - return try { - Runtime.getRuntime().exec(arrayOf("su", "-c", cmd)).waitFor() - true - } catch (e: Exception) { - false - } - } - - /** Read a root-owned file via su. */ - private fun suReadFile(path: String): String { - val p = Runtime.getRuntime().exec(arrayOf("su", "-c", "cat $path")) - val text = p.inputStream.bufferedReader().readText().trim() - p.waitFor() - return text - } - - /** Start the hexagon talker runner and connect via socket. - * If flag force_cpu_talker_gguf exists, runs with -ngl 0 (CPU fp32) to isolate - * whether the voice-cloning tremor comes from NPU fp16 drift. Diagnostic only; - * expected RTF 5-7x on CPU. */ - private fun hexStartRunner(): Boolean { - val forceCpuGguf = diagFlag("force_cpu_talker_gguf") - val nglArg = if (forceCpuGguf) "-ngl 0" else "-ngl 99 -mg 1" - nlog("Starting ${if (forceCpuGguf) "CPU GGUF" else "Hexagon"} talker runner ($nglArg)...") - val t0 = System.currentTimeMillis() - if (!suExec("pkill -f llama-tts-talker")) { nlog("su not available"); return false } - Thread.sleep(200) - if (!suExec("cd $HEX_DIR && LD_LIBRARY_PATH=. nohup ./llama-tts-talker -m /data/local/tmp/kazeia/models/talker_f16.gguf $nglArg -s $TALKER_SOCK > /data/local/tmp/kazeia/tts_runner.log 2>&1 &")) return false - // Wait for socket to be connectable (30s max — model loading takes ~15s) - for (w in 0 until 300) { - Thread.sleep(100) - try { - val sock = android.net.LocalSocket() - sock.connect(android.net.LocalSocketAddress(TALKER_SOCK, android.net.LocalSocketAddress.Namespace.FILESYSTEM)) - talkerSocket = sock - nlog("Hexagon talker connected: ${System.currentTimeMillis() - t0}ms") - return true - } catch (_: Exception) {} - } - nlog("Hexagon talker timeout (30s)") - return false - } - - /** Talker forward via socket: send embedding, get hidden+logits. */ - private fun hexForward(embeddings: List): List> { - val sock = talkerSocket ?: return emptyList() - val os = sock.outputStream - val ins = sock.inputStream - val results = mutableListOf>() - - for (emb in embeddings) { - // Send "FWRD" + embedding - os.write("FWRD".toByteArray()) - val buf = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in emb) buf.putFloat(v) - os.write(buf.array()) - os.flush() - - // Read hidden(1024*4) + logits(3072*4) = 16384 bytes - val respSize = (TALKER_DIM + TALKER_VOCAB) * 4 - val resp = ByteArray(respSize) - var read = 0 - while (read < respSize) { - val n = ins.read(resp, read, respSize - read) - if (n <= 0) break - read += n - } - val fb = java.nio.ByteBuffer.wrap(resp).order(java.nio.ByteOrder.LITTLE_ENDIAN).asFloatBuffer() - val h = FloatArray(TALKER_DIM); fb.get(h) - val l = FloatArray(TALKER_VOCAB); fb.get(l) - results.add(Pair(h, l)) - } - return results - } - - /** Reset KV cache for a new phrase via socket. */ - private fun hexReset() { - val sock = talkerSocket ?: return - sock.outputStream.write("REST".toByteArray()) - sock.outputStream.flush() - val resp = ByteArray(4) - sock.inputStream.read(resp) // blocking, waits for "OK\0\0" - } - - /** Start CP ExecuTorch runner (NPU HTP via root process). */ - private fun hexStartCpEtRunner(): Boolean { - nlog("Connecting CP ExecuTorch runner (TCP:8790)...") - val t0 = System.currentTimeMillis() - // Try connecting to existing runner first - try { - val sock = java.net.Socket() - sock.connect(java.net.InetSocketAddress("127.0.0.1", 8790), 2000) - sock.tcpNoDelay = true - cpEtSocket = sock - nlog("CP ET connected (TCP, existing): ${System.currentTimeMillis() - t0}ms") - return true - } catch (e: Exception) { - nlog("CP ET existing connection failed: ${e.message}") - } - // Start new runner - nlog("No running CP ET, starting new...") - suExec("pkill -f cp_et_runner") - Thread.sleep(200) - if (!suExec("cd /data/local/tmp/kazeia && LD_LIBRARY_PATH=qnn_libs:. ADSP_LIBRARY_PATH=qnn_libs " + - "nohup ./cp_et_runner --model_path=models/cp_transformer_fp16.pte --tcp_port=8790 " + - "> /data/local/tmp/kazeia/cp_et.log 2>&1 &")) return false - for (w in 0 until 300) { - Thread.sleep(100) - try { - val sock = java.net.Socket("127.0.0.1", 8790) - sock.tcpNoDelay = true - cpEtSocket = sock - nlog("CP ET connected (TCP): ${System.currentTimeMillis() - t0}ms") - return true - } catch (_: Exception) {} - } - nlog("CP ET timeout (30s)") - return false - } - - /** CP via ExecuTorch NPU TCP socket: send hidden+cb0_emb, recv 15 codes + timing. */ - private fun etCpForward(pastHidden: FloatArray, cb0: Int): IntArray { - val sock = cpEtSocket - if (sock == null || sock.isClosed) { nlog("CP ET socket null"); return runCpV2(pastHidden, cb0) } - try { - val os = sock.getOutputStream(); val ins = sock.getInputStream() - val buf = java.nio.ByteBuffer.allocate(2 * TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in pastHidden) buf.putFloat(v) - val cb0Emb = FloatArray(TALKER_DIM) - System.arraycopy(codecEmbedding!!, cb0.coerceIn(0, TALKER_VOCAB - 1) * TALKER_DIM, cb0Emb, 0, TALKER_DIM) - for (v in cb0Emb) buf.putFloat(v) - os.write(buf.array()); os.flush() - val resp = ByteArray(64); var read = 0 - while (read < 64) { val n = ins.read(resp, read, 64 - read); if (n <= 0) break; read += n } - if (read < 64) { nlog("CP ET read incomplete: $read/64"); return runCpV2(pastHidden, cb0) } - val codes = IntArray(15); val rb = java.nio.ByteBuffer.wrap(resp).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (i in 0 until 15) codes[i] = rb.int - return codes - } catch (e: Exception) { - nlog("CP ET error: ${e.message}"); cpEtSocket = null; useEtCp = false - return runCpV2(pastHidden, cb0) - } - } - - /** Start CP hexagon runner and connect via socket. - * NOTE: uses -ngl 0 (CPU only). Running the CP on Hexagon HTP produces deterministic - * NaN at position 2 of every request — bug in libggml-htp-v79.so for this specific - * 5-layer qwen3 graph (confirmed empirically). CPU path is ~60 ms/step and correct. - * The talker still benefits from Hexagon HMX since it runs in a separate process. */ - private fun hexStartCpRunner(): Boolean { - nlog("Starting CP runner (CPU, -ngl 0 to avoid HMX NaN bug)...") - val t0 = System.currentTimeMillis() - if (!suExec("pkill -f llama-tts-cp")) { nlog("su not available for CP"); return false } - Thread.sleep(200) - if (!suExec("cd $HEX_DIR && LD_LIBRARY_PATH=. nohup ./llama-tts-cp -m /data/local/tmp/kazeia/models/cp_f16.gguf -ngl 0 -s $CP_SOCK > /data/local/tmp/kazeia/cp_runner.log 2>&1 &")) return false - for (w in 0 until 300) { - Thread.sleep(100) - try { - val sock = android.net.LocalSocket() - sock.connect(android.net.LocalSocketAddress(CP_SOCK, android.net.LocalSocketAddress.Namespace.FILESYSTEM)) - cpSocket = sock - nlog("CP Hexagon connected: ${System.currentTimeMillis() - t0}ms") - return true - } catch (_: Exception) {} - } - nlog("CP Hexagon timeout (30s)") - return false - } - - /** CP via Hexagon NPU socket: send hidden+cb0_emb, get 15 codes. */ - private fun hexCpForward(pastHidden: FloatArray, cb0: Int): IntArray { - val sock = cpSocket - if (sock == null) { - nlog("CP socket NULL, falling back to CPU") - return runCpCpu(pastHidden, cb0) - } - try { - val os = sock.outputStream - val ins = sock.inputStream - - val t0 = System.currentTimeMillis() - // Send hidden(1024) + cb0_emb(1024) = 8192 bytes - val buf = java.nio.ByteBuffer.allocate(2 * TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in pastHidden) buf.putFloat(v) - val cb0Emb = FloatArray(TALKER_DIM) - System.arraycopy(codecEmbedding!!, cb0.coerceIn(0, TALKER_VOCAB - 1) * TALKER_DIM, cb0Emb, 0, TALKER_DIM) - for (v in cb0Emb) buf.putFloat(v) - os.write(buf.array()); os.flush() - val writeMs = System.currentTimeMillis() - t0 - - // Read 15 codes (60 bytes) + timing (4 bytes) = 64 bytes - val resp = ByteArray(64) - var read = 0 - while (read < 64) { val n = ins.read(resp, read, 64 - read); if (n <= 0) break; read += n } - val totalMs = System.currentTimeMillis() - t0 - - if (cpCallCount <= 3) nlog("CP socket: write=${writeMs}ms, total=${totalMs}ms, read=$read/64") - - if (read < 64) { - nlog("CP socket read incomplete: $read/64 bytes") - return runCpCpu(pastHidden, cb0) - } - - val codes = IntArray(15) - val rb = java.nio.ByteBuffer.wrap(resp).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (i in 0 until 15) codes[i] = rb.int - // Defensive clamp: CP on first call may occasionally return uninitialised - // memory before warm-up stabilises. Out-of-range codes would crash BigVGAN - // decoder (ArrayIndexOutOfBoundsException on codebook lookup). Map anything - // invalid to 0 — a single-frame artefact is much better than an app crash. - var n_bad = 0 - for (i in 0 until 15) { - if (codes[i] < 0 || codes[i] >= CODEBOOK_SIZE) { codes[i] = 0; n_bad++ } - } - if (n_bad > 0) nlog("CP socket: clamped $n_bad out-of-range codes to 0") - return codes - } catch (e: Exception) { - nlog("CP socket error: ${e.message}, falling back to CPU") - cpSocket = null - return runCpCpu(pastHidden, cb0) - } - } - - /** Ensure hexagon runners are alive. Only restarts if they're not connected. */ - private fun ensureHexagonRunners() { - if (!useHexagonTalker) { - val hexRunner = java.io.File("/data/local/tmp/kazeia/llama-hex/llama-tts-talker") - val hexModel = java.io.File("/data/local/tmp/kazeia/models/talker_f16.gguf") - if (hexRunner.exists() && hexModel.exists() && hexStartRunner()) { - useHexagonTalker = true - } - } - if (!useHexagonCp) { - val hexCpRunner = java.io.File("/data/local/tmp/kazeia/llama-hex/llama-tts-cp") - val hexCpModel = java.io.File("/data/local/tmp/kazeia/models/cp_f16.gguf") - if (hexCpRunner.exists() && hexCpModel.exists() && hexStartCpRunner()) { - useHexagonCp = true - } - } - } - - /** Stop all runners and release DSP for QNN decode. Runners restart on next generate(). */ - private fun hexStopRunner() { - try { - talkerSocket?.outputStream?.write("QUIT".toByteArray()) - talkerSocket?.close() - cpSocket?.close() - cpEtSocket?.close() - Thread.sleep(300) - suExec("pkill -f llama-tts") - suExec("pkill -f cp_et_runner") - Thread.sleep(200) - } catch (_: Exception) {} - talkerSocket = null; cpSocket = null; cpEtSocket = null - useHexagonTalker = false; useHexagonCp = false; useEtCp = false - nlog("Hexagon runners stopped, DSP released") - } - - /** Full interleaved generation using Hexagon NPU talker + CPU CP. */ - /** All-NPU pipeline: talker .pte + CP .pte via JNI, no root. */ - private fun runInterleavedPte(textEmbedsList: List, maxGenTokens: Int): Array { - val talkerMod = talkerPteModule!! - val cpMod = cpPteModule!! - val tCos = talkerPteRotaryCos!! - val tSin = talkerPteRotarySin!! - val cCos = cpRotaryCos ?: return emptyArray() - val cSin = cpRotarySin ?: return emptyArray() - val eosE = ttsEosEmbed ?: return emptyArray() - val padE = ttsPadEmbed ?: return emptyArray() - - val prefill = buildPrefillEmbeddings(textEmbedsList) - if (prefill.isEmpty()) return emptyArray() - val trailingEmbeds = if (textEmbedsList.size > 1) textEmbedsList.subList(1, textEmbedsList.size) else emptyList() - var trailingIdx = 0 - - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - - // Talker KV caches [TALKER_LAYERS × (k,v)] each [1, 8, TALKER_PTE_KV_LEN, 128] - val tkvSize = TALKER_HEADS * TALKER_PTE_KV_LEN * TALKER_HEAD_DIM - var tK = Array(TALKER_LAYERS) { FloatArray(tkvSize) } - var tV = Array(TALKER_LAYERS) { FloatArray(tkvSize) } - val maskData = FloatArray(TALKER_PTE_KV_LEN) { -1e9f } - - var pos = 0; var currentCb0 = -1; var pastHidden: FloatArray? = null - - nlog("PTE pipeline: prefill=${prefill.size}, trailing=${trailingEmbeds.size}") - - // Capture mode: save all talker inputs for reuse with C++ pipeline - val capturedEmbeds = mutableListOf() - - // ===== PREFILL ===== - val tPrefill = System.currentTimeMillis() - for (step in prefill.indices) { - capturedEmbeds.add(prefill[step].clone()) // capture prefill input - val maskIdx = TALKER_PTE_KV_LEN - 1 - minOf(pos, TALKER_PTE_KV_LEN - 1) - if (maskIdx >= 0) maskData[maskIdx] = 0f - - val cosSlice = FloatArray(TALKER_HEAD_DIM) - System.arraycopy(tCos, pos * TALKER_HEAD_DIM, cosSlice, 0, TALKER_HEAD_DIM) - val sinSlice = FloatArray(TALKER_HEAD_DIM) - System.arraycopy(tSin, pos * TALKER_HEAD_DIM, sinSlice, 0, TALKER_HEAD_DIM) - - val inputs = mutableListOf( - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(prefill[step], longArrayOf(1, 1, TALKER_DIM.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(maskData.clone(), longArrayOf(1, 1, 1, TALKER_PTE_KV_LEN.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(cosSlice, longArrayOf(1, 1, TALKER_HEAD_DIM.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(sinSlice, longArrayOf(1, 1, TALKER_HEAD_DIM.toLong()))) - ) - for (i in 0 until TALKER_LAYERS) { - inputs.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(tK[i], longArrayOf(1, TALKER_HEADS.toLong(), TALKER_PTE_KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())))) - inputs.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(tV[i], longArrayOf(1, TALKER_HEADS.toLong(), TALKER_PTE_KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())))) - } - - val out = talkerMod.forward(*inputs.toTypedArray()) - pastHidden = out[0].toTensor().dataAsFloatArray - val logits = out[1].toTensor().dataAsFloatArray - for (i in 0 until TALKER_LAYERS) { - tK[i] = out[2 + i * 2].toTensor().dataAsFloatArray - tV[i] = out[3 + i * 2].toTensor().dataAsFloatArray - } - pos++ - - if (step == 0 && File("/data/local/tmp/kazeia/dump_compare").exists()) { - try { - java.io.FileOutputStream("/data/local/tmp/kazeia/pte_input_step0.bin").use { fos -> - val bb = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in prefill[0]) bb.putFloat(v); fos.write(bb.array()) - } - java.io.FileOutputStream("/data/local/tmp/kazeia/pte_hidden_step0.bin").use { fos -> - val bb = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in pastHidden!!) bb.putFloat(v); fos.write(bb.array()) - } - File("/data/local/tmp/kazeia/pte_input_step0.bin").setReadable(true, false) - File("/data/local/tmp/kazeia/pte_hidden_step0.bin").setReadable(true, false) - val h = pastHidden!! - var hMin = h[0]; var hMax = h[0]; var hSum = 0.0 - for (v in h) { if (v < hMin) hMin = v; if (v > hMax) hMax = v; hSum += v } - nlog("DUMP_CMP pte step0 hidden: min=$hMin max=$hMax mean=${hSum/h.size}") - } catch (e: Exception) { nlog("dump_compare pte failed: ${e.message}") } - } - - if (step == prefill.size - 1) { - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - currentCb0 = sampleTopK(logits, 0.9f, 50) - } - } - nlog("Prefill (PTE): ${System.currentTimeMillis() - tPrefill}ms, ${prefill.size} steps") - - if (currentCb0 < 0 || currentCb0 == CODEC_EOS) return emptyArray() - - // ===== INTERLEAVED GENERATION ===== - var totalTalkerMs = 0L; var totalCpMs = 0L - for (genStep in 0 until maxGenTokens) { - val codes = IntArray(NUM_CODEBOOKS); codes[0] = currentCb0 - - // 1. CP: predict CB1-15 - val tCp = System.currentTimeMillis() - val cpCodes = runCpPte(pastHidden!!, currentCb0) - totalCpMs += System.currentTimeMillis() - tCp - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes); generatedCb0.add(currentCb0) - - if (genStep < 3) nlog("Step ${genStep+1}: cb0=$currentCb0 cb1=${codes[1]}") - - // 2. Build next talker input - val codecSum = FloatArray(TALKER_DIM) - addEmb(codecSum, codecEmb(codes[0])) - for (cb in 1 until NUM_CODEBOOKS) addEmb(codecSum, cpEmb(cb - 1, codes[cb])) - - val nextEmbed: FloatArray = when { - trailingIdx < trailingEmbeds.size -> { trailingIdx++; sumEmb(codecSum, trailingEmbeds[trailingIdx - 1]) } - trailingIdx == trailingEmbeds.size -> { trailingIdx++; sumEmb(codecSum, eosE) } - else -> sumEmb(codecSum, padE) - } - - capturedEmbeds.add(nextEmbed.clone()) // capture decode input - - // 3. Talker step - val maskIdx = TALKER_PTE_KV_LEN - 1 - minOf(pos, TALKER_PTE_KV_LEN - 1) - if (maskIdx >= 0) maskData[maskIdx] = 0f - - val cosSlice = FloatArray(TALKER_HEAD_DIM) - System.arraycopy(tCos, minOf(pos, tCos.size / TALKER_HEAD_DIM - 1) * TALKER_HEAD_DIM, cosSlice, 0, TALKER_HEAD_DIM) - val sinSlice = FloatArray(TALKER_HEAD_DIM) - System.arraycopy(tSin, minOf(pos, tSin.size / TALKER_HEAD_DIM - 1) * TALKER_HEAD_DIM, sinSlice, 0, TALKER_HEAD_DIM) - - val inputs = mutableListOf( - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(nextEmbed, longArrayOf(1, 1, TALKER_DIM.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(maskData.clone(), longArrayOf(1, 1, 1, TALKER_PTE_KV_LEN.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(cosSlice, longArrayOf(1, 1, TALKER_HEAD_DIM.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(sinSlice, longArrayOf(1, 1, TALKER_HEAD_DIM.toLong()))) - ) - for (i in 0 until TALKER_LAYERS) { - inputs.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(tK[i], longArrayOf(1, TALKER_HEADS.toLong(), TALKER_PTE_KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())))) - inputs.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(tV[i], longArrayOf(1, TALKER_HEADS.toLong(), TALKER_PTE_KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())))) - } - - val tTalker = System.currentTimeMillis() - val out = talkerMod.forward(*inputs.toTypedArray()) - totalTalkerMs += System.currentTimeMillis() - tTalker - - pastHidden = out[0].toTensor().dataAsFloatArray - val logits = out[1].toTensor().dataAsFloatArray - for (i in 0 until TALKER_LAYERS) { - tK[i] = out[2 + i * 2].toTensor().dataAsFloatArray - tV[i] = out[3 + i * 2].toTensor().dataAsFloatArray - } - pos++ - - // 4. Next CB0 - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - val nextCb0 = sampleTopK(logits, 0.9f, 50) - - if (nextCb0 == CODEC_EOS) { nlog("EOS at gen step ${genStep + 2}"); break } - if (generatedCb0.size >= 9 && generatedCb0.takeLast(9).all { it == nextCb0 }) { - nlog("Degeneration at step ${genStep + 2}"); break - } - currentCb0 = nextCb0 - } - - val n = allCodes.size - nlog("Generated $n tokens | Talker(PTE): ${totalTalkerMs}ms (${totalTalkerMs/maxOf(n,1)}ms/step) | CP(PTE): ${totalCpMs}ms (${totalCpMs/maxOf(n,1)}ms/step)") - - // Save captured embeds for C++ pipeline reuse - if (capturedEmbeds.isNotEmpty()) { - try { - val capPath = "/data/local/tmp/kazeia/captured_embeds.bin" - val nPrefill = prefill.size - val fos = java.io.FileOutputStream(capPath) - val hdr = java.nio.ByteBuffer.allocate(8).order(java.nio.ByteOrder.LITTLE_ENDIAN) - hdr.putInt(nPrefill); hdr.putInt(capturedEmbeds.size) - fos.write(hdr.array()) - for (emb in capturedEmbeds) { - val buf = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in emb) buf.putFloat(v) - fos.write(buf.array()) - } - fos.close() - nlog("Captured ${capturedEmbeds.size} embeds → $capPath") - } catch (e: Exception) { nlog("Capture save failed: ${e.message}") } - } - - return allCodes.toTypedArray() - } - - private fun runInterleavedHexagon(textEmbedsList: List, maxGenTokens: Int): Array { - val eosE = ttsEosEmbed ?: return emptyArray() - val padE = ttsPadEmbed ?: return emptyArray() - - val prefill = buildPrefillEmbeddings(textEmbedsList) - if (prefill.isEmpty()) return emptyArray() - val trailingEmbeds = if (textEmbedsList.size > 1) textEmbedsList.subList(1, textEmbedsList.size) else emptyList() - var trailingIdx = 0 - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - - // Reset KV cache for new phrase (runner already started at load time) - hexReset() - - var totalTalkerMs = 0L; var totalCpMs = 0L - - try { - // Prefill - val tPrefill = System.currentTimeMillis() - val prefillResults = hexForward(prefill) - val prefillMs = System.currentTimeMillis() - tPrefill - nlog("Prefill (Hexagon): ${prefillMs}ms, ${prefillResults.size} steps") - - var pastHidden = prefillResults.last().first - val prefillLogits = prefillResults.last().second - // Suppress non-codec tokens - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) prefillLogits[j] = Float.NEGATIVE_INFINITY } - var currentCb0 = sampleTopK(prefillLogits, 0.9f, 50) - nlog("Prefill done: first cb0=$currentCb0") - - // Generation loop - for (genStep in 0 until maxGenTokens) { - // CP on CPU - val tCp = System.currentTimeMillis() - val codes = IntArray(NUM_CODEBOOKS) - codes[0] = currentCb0 - val cpCodes = runCodePredictorInterleaved(pastHidden, currentCb0) - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes) - generatedCb0.add(currentCb0) - val cpMs = System.currentTimeMillis() - tCp - totalCpMs += cpMs - - if (genStep < 3) nlog("Gen step ${genStep + 1}: cb0=$currentCb0 cb1=${codes[1]} [CP=${cpMs}ms]") - - // Build next embedding - val codecSum = FloatArray(TALKER_DIM) - addEmb(codecSum, codecEmb(codes[0])) - for (cb in 1 until NUM_CODEBOOKS) addEmb(codecSum, cpEmb(cb - 1, codes[cb])) - - val nextEmbed: FloatArray = if (trailingIdx < trailingEmbeds.size) { - sumEmb(codecSum, trailingEmbeds[trailingIdx++]) - } else if (trailingIdx == trailingEmbeds.size) { - trailingIdx++; sumEmb(codecSum, eosE) - } else { - sumEmb(codecSum, padE) - } - - // Talker forward on Hexagon NPU - val tTalker = System.currentTimeMillis() - val results = hexForward(listOf(nextEmbed)) - val talkerMs = System.currentTimeMillis() - tTalker - totalTalkerMs += talkerMs - if (genStep < 3) nlog(" Talker(HEX)=${talkerMs}ms") - - pastHidden = results[0].first - val logits = results[0].second - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - val nextCb0 = sampleTopK(logits, 0.9f, 50) - - if (nextCb0 == CODEC_EOS) { nlog("EOS at gen step ${genStep + 2}"); break } - if (generatedCb0.size >= 9 && generatedCb0.takeLast(9).all { it == nextCb0 }) { - nlog("Degeneration at step ${genStep + 2}"); break - } - currentCb0 = nextCb0 - } - } finally { - // Runners stay alive here — hexStopRunner() is called before QNN decode - } - - val n = allCodes.size - nlog("Generated $n tokens | Talker(HEX): ${totalTalkerMs}ms (${totalTalkerMs/maxOf(n,1)}ms/step) | CP: ${totalCpMs}ms (${totalCpMs/maxOf(n,1)}ms/step)") - return allCodes.toTypedArray() - } - - /** - * Talker + CP both via libtts_talker_cpu.so / libtts_cp_cpu.so (llama.cpp). - * Dynamic KV (no fixed length, no mask juggling, no manual RoPE slicing) — - * eliminates the chunk-transition "beg beg beg" artefact rooted in the - * .pte fixed-KV layout. Each forward call hands llama.cpp an [1024] embd - * and gets back [hidden(1024) || logits(3072)]. - */ - private fun runInterleavedCpu(textEmbedsList: List, maxGenTokens: Int): Array { - val prefill = buildPrefillEmbeddings(textEmbedsList) - if (prefill.isEmpty()) return emptyArray() - val trailingEmbeds = if (textEmbedsList.size > 1) textEmbedsList.subList(1, textEmbedsList.size) else emptyList() - return runInterleavedCpuFromEmbeds(prefill, trailingEmbeds, maxGenTokens) - } - - /** - * CPU equivalent of [runInterleavedPteFromEmbeds]: takes pre-built prefill - * + trailing embeddings (e.g. from Stage-2 voice-cloning prefix/suffix) and - * runs the interleaved Talker+CP loop via the ggml-cpu JNI handles. - */ - private fun runInterleavedCpuFromEmbeds( - prefill: List, - trailingEmbeds: List, - maxGenTokens: Int, - onCodeStep: ((step: Int, codes: IntArray) -> Unit)? = null, - // Étage minimal avant d'armer le boost EOS dynamique. Sous ce seuil, le - // talker peut babiller sans être forcé. Au-delà, on surveille le rang - // de CODEC_EOS et on l'amplifie quand le modèle « pense à terminer » - // (cf. runHexGenWithPrefill). Empêche le talker de se faire couper - // brutalement à maxGen, ce qui produisait « page beg beg » à la fin. - eosBoostMinStep: Int = -1, - // ggml-cpu donne un rang EOS naturellement plus haut que le Hex - // (100-300 vs 50-60). Seuil revu vers le haut sinon le boost ne - // s'arme jamais et on retombe sur le hard fallback brutal. - eosRankTrigger: Int = 300, - eosBoostScale: Float = 4.0f, - ): Array { - val tHandle = talkerCpuHandle - val cHandle = cpCpuHandle - if (tHandle == 0L || cHandle == 0L) return emptyArray() - val eosE = ttsEosEmbed ?: return emptyArray() - val padE = ttsPadEmbed ?: return emptyArray() - if (prefill.isEmpty()) return emptyArray() - var trailingIdx = 0 - - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - - // Fresh KV for this sentence - TtsTalkerCpuJni.nativeReset(tHandle) - TtsCpCpuJni.nativeReset(cHandle) - - var pastHidden: FloatArray? = null - var currentCb0 = -1 - - // ===== PREFILL ===== - val tPrefill = System.currentTimeMillis() - for (step in prefill.indices) { - val out = TtsTalkerCpuJni.nativeForward(tHandle, prefill[step]) ?: run { - nlog("Talker (ggml-cpu) forward returned null at prefill step $step") - return emptyArray() - } - if (out.size != TALKER_DIM + TALKER_VOCAB) { - nlog("Talker (ggml-cpu) bad output size ${out.size} (expected ${TALKER_DIM + TALKER_VOCAB})") - return emptyArray() - } - if (step == 0 && File("/data/local/tmp/kazeia/dump_compare").exists()) { - try { - java.io.FileOutputStream("/data/local/tmp/kazeia/cpu_input_step0.bin").use { fos -> - val bb = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in prefill[0]) bb.putFloat(v); fos.write(bb.array()) - } - java.io.FileOutputStream("/data/local/tmp/kazeia/cpu_hidden_step0.bin").use { fos -> - val bb = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (i in 0 until TALKER_DIM) bb.putFloat(out[i]); fos.write(bb.array()) - } - File("/data/local/tmp/kazeia/cpu_input_step0.bin").setReadable(true, false) - File("/data/local/tmp/kazeia/cpu_hidden_step0.bin").setReadable(true, false) - var hMin = out[0]; var hMax = out[0]; var hSum = 0.0 - for (i in 0 until TALKER_DIM) { val v = out[i]; if (v < hMin) hMin = v; if (v > hMax) hMax = v; hSum += v } - nlog("DUMP_CMP cpu step0 hidden: min=$hMin max=$hMax mean=${hSum/TALKER_DIM}") - } catch (e: Exception) { nlog("dump_compare cpu failed: ${e.message}") } - } - if (step == prefill.size - 1) { - pastHidden = FloatArray(TALKER_DIM); System.arraycopy(out, 0, pastHidden!!, 0, TALKER_DIM) - val logits = FloatArray(TALKER_VOCAB); System.arraycopy(out, TALKER_DIM, logits, 0, TALKER_VOCAB) - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - currentCb0 = sampleTopK(logits, 0.9f, 200) - } - } - nlog("Prefill (ggml-cpu): ${System.currentTimeMillis() - tPrefill}ms, ${prefill.size} steps") - - if (currentCb0 < 0 || currentCb0 == CODEC_EOS) return emptyArray() - - // ===== INTERLEAVED GENERATION ===== - var totalTalkerMs = 0L; var totalCpMs = 0L - var boostStepsActive = 0 - var consecLowRank = 0 - // Hard fallback : si le talker n'a pas armé l'EOS boost naturel après - // 3 × eosBoostMinStep steps (= 1.5 × expectedSteps), on FORCE l'arming. - // Donne assez de marge pour que le talker termine la phrase, mais - // empêche le babillage indéfini. - val hardBoostStep = if (eosBoostMinStep >= 0) eosBoostMinStep * 3 else Int.MAX_VALUE - for (genStep in 0 until maxGenTokens) { - val codes = IntArray(NUM_CODEBOOKS); codes[0] = currentCb0 - - // 1. CP: predict CB1-15 in one JNI call (17 fwd + 15 NEON argmax inside) - val tCp = System.currentTimeMillis() - val cb0Emb = FloatArray(TALKER_DIM) - System.arraycopy(codecEmbedding!!, currentCb0.coerceIn(0, TALKER_VOCAB - 1) * TALKER_DIM, cb0Emb, 0, TALKER_DIM) - val cpCodes = TtsCpCpuJni.nativeForward(cHandle, pastHidden!!, cb0Emb) - totalCpMs += System.currentTimeMillis() - tCp - if (cpCodes == null || cpCodes.size != NUM_CODEBOOKS - 1) { - nlog("CP (ggml-cpu) forward null/bad at gen step $genStep — aborting") - break - } - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes); generatedCb0.add(currentCb0) - onCodeStep?.invoke(genStep, codes) - - if (genStep < 3) nlog("Step ${genStep + 1}: cb0=$currentCb0 cb1=${codes[1]}") - - // 2. Build next talker input - val codecSum = FloatArray(TALKER_DIM) - addEmb(codecSum, codecEmb(codes[0])) - for (cb in 1 until NUM_CODEBOOKS) addEmb(codecSum, cpEmb(cb - 1, codes[cb])) - - val nextEmbed: FloatArray = when { - trailingIdx < trailingEmbeds.size -> { trailingIdx++; sumEmb(codecSum, trailingEmbeds[trailingIdx - 1]) } - trailingIdx == trailingEmbeds.size -> { trailingIdx++; sumEmb(codecSum, eosE) } - else -> sumEmb(codecSum, padE) - } - - // 3. Talker step - val tTalker = System.currentTimeMillis() - val out = TtsTalkerCpuJni.nativeForward(tHandle, nextEmbed) - totalTalkerMs += System.currentTimeMillis() - tTalker - if (out == null) { - nlog("Talker (ggml-cpu) forward null at gen step $genStep — aborting") - break - } - - pastHidden = FloatArray(TALKER_DIM); System.arraycopy(out, 0, pastHidden!!, 0, TALKER_DIM) - val logits = FloatArray(TALKER_VOCAB); System.arraycopy(out, TALKER_DIM, logits, 0, TALKER_VOCAB) - - // 4. Next CB0 — restreindre au codebook + EOS, anti-bias seen - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - - // EOS boost dynamique (cf. runHexGenWithPrefill). Sans ça, le talker - // ne produit pas spontanément CODEC_EOS sur les phrases longues et - // se fait couper brutalement à maxGen → fin abrupte audible. - if (eosBoostMinStep >= 0) { - val eosLogit0 = logits[CODEC_EOS] - var eosRank = 0 - for (j in logits.indices) if (logits[j] > eosLogit0) eosRank++ - if (genStep % 20 == 0) nlog("step ${genStep+1}: eosRank=$eosRank eosLogit=${"%.2f".format(eosLogit0)}") - if (boostStepsActive == 0 && genStep >= eosBoostMinStep) { - if (eosRank < eosRankTrigger) { - consecLowRank++ - if (consecLowRank >= 3) { - nlog("CPU EOS boost armed at step ${genStep+1} (rank=$eosRank, 3 consecutive low)") - boostStepsActive = 1 - } - } else { - consecLowRank = 0 - } - // Fallback : si on dépasse 2×eosBoostMinStep sans armer - // naturel, on FORCE l'arming pour éviter le babillage. - if (boostStepsActive == 0 && genStep >= hardBoostStep) { - nlog("CPU EOS boost FORCED at step ${genStep+1} (rank=$eosRank, hard fallback)") - boostStepsActive = 1 - } - } else if (boostStepsActive > 0) { - boostStepsActive++ - } - if (boostStepsActive > 0) { - logits[CODEC_EOS] += boostStepsActive * eosBoostScale - } - // Argmax check : si EOS devient le top, on stoppe immédiatement - var argmax = 0; var argmaxV = logits[0] - for (j in 1 until logits.size) if (logits[j] > argmaxV) { argmaxV = logits[j]; argmax = j } - if (argmax == CODEC_EOS) { - nlog("CPU EOS (boosted argmax) at step ${genStep + 1}") - break - } - } - - val nextCb0 = sampleTopK(logits, 0.9f, 200) - if (nextCb0 == CODEC_EOS) { nlog("CPU EOS (sampled) at step ${genStep + 2}"); break } - if (generatedCb0.size >= 9 && generatedCb0.takeLast(9).all { it == nextCb0 }) { - nlog("Degeneration at step ${genStep + 2}"); break - } - currentCb0 = nextCb0 - } - - val n = allCodes.size - nlog("Generated $n tokens | Talker(ggml-cpu): ${totalTalkerMs}ms (${totalTalkerMs/maxOf(n,1)}ms/step) | CP(ggml-cpu): ${totalCpMs}ms (${totalCpMs/maxOf(n,1)}ms/step)") - if (File("/data/local/tmp/kazeia/dump_codes").exists()) { - try { - val ts = System.currentTimeMillis() - val path = "/data/local/tmp/kazeia/tts_dump/codes_$ts.bin" - java.io.FileOutputStream(path).use { fos -> - val hdr = java.nio.ByteBuffer.allocate(8).order(java.nio.ByteOrder.LITTLE_ENDIAN) - hdr.putInt(n); hdr.putInt(NUM_CODEBOOKS); fos.write(hdr.array()) - val buf = java.nio.ByteBuffer.allocate(n * NUM_CODEBOOKS * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (codes in allCodes) for (v in codes) buf.putInt(v); fos.write(buf.array()) - } - File(path).setReadable(true, false) - nlog("Codes dump : $path ($n × $NUM_CODEBOOKS)") - } catch (e: Exception) { nlog("dump_codes failed: ${e.message}") } - } - return allCodes.toTypedArray() - } - - /** - * Run interleaved talker + code predictor pipeline. - * - * At each step: - * 1. Talker forward → logits + hidden_state - * 2. Code predictor (hidden, cb0_emb) → CB1-15 autoregressively - * 3. Sum all 16 codebook embeddings + trailing text → next talker input - * - * Returns all 16 codebooks as [numTokens][16]. - */ - private fun runInterleavedGeneration(textEmbedsList: List, maxGenTokens: Int = 50): Array { - // Priority 0: ggml-cpu Talker+CP (llama.cpp dynamic KV) when active - if (useCpuPath) { - return runInterleavedCpu(textEmbedsList, maxGenTokens) - } - // Priority 1: All .pte JNI on NPU (no root needed) - if (talkerPteModule != null && cpPteModule != null) { - return runInterleavedPte(textEmbedsList, maxGenTokens) - } - // Priority 2: Hexagon talker + socket/ONNX CP - ensureHexagonRunners() - if (useHexagonTalker) { - return runInterleavedHexagon(textEmbedsList, maxGenTokens) - } - val env = ortEnv ?: return emptyArray() - if (talkerKv == null) return emptyArray() - val session = talkerKv // may be null if using NPU - val eosE = ttsEosEmbed ?: return emptyArray() - val padE = ttsPadEmbed ?: return emptyArray() - - val prefill = buildPrefillEmbeddings(textEmbedsList) - if (prefill.isEmpty()) return emptyArray() - - // Trailing text embeddings (all except first, which is in prefill) - val trailingEmbeds = if (textEmbedsList.size > 1) textEmbedsList.subList(1, textEmbedsList.size) else emptyList() - var trailingIdx = 0 - - val allCodes = mutableListOf() // each entry is [16] codebooks for one time step - val generatedCb0 = mutableListOf() - - val kCacheSize = TALKER_HEADS * TALKER_HEAD_DIM * KV_LEN - val vCacheSize = TALKER_HEADS * KV_LEN * TALKER_HEAD_DIM - var kCaches = Array(TALKER_LAYERS) { FloatArray(kCacheSize) } - var vCaches = Array(TALKER_LAYERS) { FloatArray(vCacheSize) } - val maskData = FloatArray(MAX_CONTEXT) { MASK_NEG } - - var pos = 0 - var currentCb0 = -1 - var pastHidden: FloatArray? = null - - // ===== PREFILL ===== - for (step in prefill.indices) { - maskData[MAX_CONTEXT - 1 - step] = 0f - val res = runTalkerStep(env, session!!, prefill[step], maskData, pos, kCaches, vCaches) - kCaches = res.newK; vCaches = res.newV; pastHidden = res.hidden - pos++ - - if (step == prefill.size - 1) { - // Apply suppression + sampling to get first codec_0 - val logits = res.logits - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - currentCb0 = sampleTopK(logits, 0.9f, 50) - nlog("Prefill done: first cb0=$currentCb0") - } - } - - if (currentCb0 < 0 || currentCb0 == CODEC_EOS) return emptyArray() - - // ===== INTERLEAVED GENERATION ===== - var totalTalkerMs = 0L; var totalCpMs = 0L - for (genStep in 0 until maxGenTokens) { - // 1. Run code predictor: (pastHidden, cb0_emb) → CB1-15 - val codes = IntArray(NUM_CODEBOOKS) - codes[0] = currentCb0 - - val tCp = System.currentTimeMillis() - val cpCodes = runCodePredictorInterleaved(pastHidden!!, currentCb0) - val cpMs = System.currentTimeMillis() - tCp - totalCpMs += cpMs - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes) - generatedCb0.add(currentCb0) - - if (genStep < 3) { - nlog("Gen step ${genStep + 1}: cb0=$currentCb0 cb1=${codes[1]} [CP=${cpMs}ms]") - } - - // 2. Build next talker input: sum of ALL 16 codebook embeddings + trailing text - val codecSum = FloatArray(TALKER_DIM) - addEmb(codecSum, codecEmb(codes[0])) - for (cb in 1 until NUM_CODEBOOKS) { - addEmb(codecSum, cpEmb(cb - 1, codes[cb])) - } - - // Text side: trailing text tokens, then tts_eos, then tts_pad (NOT nothing!) - // Python model expects tts_pad after text exhaustion - crucial for EOS convergence - val nextEmbed: FloatArray - if (trailingIdx < trailingEmbeds.size) { - nextEmbed = sumEmb(codecSum, trailingEmbeds[trailingIdx]) - trailingIdx++ - } else if (trailingIdx == trailingEmbeds.size) { - nextEmbed = sumEmb(codecSum, eosE) - trailingIdx++ - } else { - nextEmbed = sumEmb(codecSum, padE) - } - - // 3. Run talker step - maskData[MAX_CONTEXT - 1 - pos] = 0f - val tTalker = System.currentTimeMillis() - val res = runTalkerStep(env, session!!, nextEmbed, maskData, pos, kCaches, vCaches) - val talkerMs = System.currentTimeMillis() - tTalker - totalTalkerMs += talkerMs - kCaches = res.newK; vCaches = res.newV; pastHidden = res.hidden - pos++ - - if (genStep < 3) { - nlog(" Talker=${talkerMs}ms") - } - - // 4. Get next cb0 from logits - val logits = res.logits - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - // HuggingFace repetition penalty: 1.05x once per unique token in history - val seen = HashSet() - for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { - logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f - } - val nextCb0 = sampleTopK(logits, 0.9f, 50) - - if (nextCb0 == CODEC_EOS) { - nlog("EOS at gen step ${genStep + 2}") - break - } - // Safety: stop on extreme repetition (10+ identical = model degeneration) - if (generatedCb0.size >= 9) { - val last9 = generatedCb0.takeLast(9) - if (last9.all { it == nextCb0 }) { - nlog("Degeneration at step ${genStep + 2} (token $nextCb0 × 10), stopping") - break - } - } - currentCb0 = nextCb0 - } - - val n = allCodes.size - nlog("Generated $n tokens | Talker: ${totalTalkerMs}ms (${totalTalkerMs/maxOf(n,1)}ms/step) | CP: ${totalCpMs}ms (${totalCpMs/maxOf(n,1)}ms/step)") - return allCodes.toTypedArray() - } - - /** Single talker step: returns logits, hidden, and updated KV caches */ - private data class TalkerStepResult( - val logits: FloatArray, val hidden: FloatArray, - val newK: Array, val newV: Array - ) - - private fun runTalkerStep( - env: OrtEnvironment, session: OrtSession, - inputEmbed: FloatArray, maskData: FloatArray, pos: Int, - kCaches: Array, vCaches: Array - ): TalkerStepResult { - if (talkerUsesCosSin) { - return runTalkerStepMRoPE(env, session, inputEmbed, maskData, pos, kCaches, vCaches) - } - val embedTensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(inputEmbed), longArrayOf(1, 1, TALKER_DIM.toLong())) - val maskTensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(maskData.clone()), longArrayOf(1, 1, 1, MAX_CONTEXT.toLong())) - val posTensor = if (talkerUsesInt64Pos) { - OnnxTensor.createTensor(env, java.nio.LongBuffer.wrap(longArrayOf(pos.toLong())), longArrayOf(1)) - } else { - OnnxTensor.createTensor(env, IntBuffer.wrap(intArrayOf(pos)), longArrayOf(1)) - } - - val inputs = LinkedHashMap() - inputs["inputs_embeds"] = embedTensor - inputs["attention_mask"] = maskTensor - for (i in 0 until TALKER_LAYERS) { - inputs["k_${i}_in"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(kCaches[i]), - longArrayOf(1, TALKER_HEADS.toLong(), TALKER_HEAD_DIM.toLong(), KV_LEN.toLong())) - inputs["v_${i}_in"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(vCaches[i]), - longArrayOf(1, TALKER_HEADS.toLong(), KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())) - } - inputs["position_ids"] = posTensor - - val result = session.run(inputs) - - // logits [1, 3072, 1, 1] - val logits = FloatArray(TALKER_VOCAB) - (result.get(0) as OnnxTensor).floatBuffer.get(logits) - - // hidden [1, 1, 1024] at output index 57 - val hidden = FloatArray(TALKER_DIM) - (result.get(57) as OnnxTensor).floatBuffer.get(hidden) - - // KV caches - val kCacheSize = TALKER_HEADS * TALKER_HEAD_DIM * KV_LEN - val vCacheSize = TALKER_HEADS * KV_LEN * TALKER_HEAD_DIM - val newK = Array(TALKER_LAYERS) { FloatArray(kCacheSize) } - val newV = Array(TALKER_LAYERS) { FloatArray(vCacheSize) } - for (i in 0 until TALKER_LAYERS) { - (result.get(1 + i * 2) as OnnxTensor).floatBuffer.get(newK[i]) - (result.get(2 + i * 2) as OnnxTensor).floatBuffer.get(newV[i]) - } - - for ((_, v) in inputs) v.close() - result.close() - - return TalkerStepResult(logits, hidden, newK, newV) - } - - /** New talker step with M-RoPE: cos/sin inputs, KV shape [1,8,199,128]. */ - private fun runTalkerStepMRoPE( - env: OrtEnvironment, session: OrtSession, - inputEmbed: FloatArray, maskData: FloatArray, pos: Int, - kCaches: Array, vCaches: Array - ): TalkerStepResult { - val cos = rotaryCos ?: return TalkerStepResult(FloatArray(TALKER_VOCAB), FloatArray(TALKER_DIM), kCaches, vCaches) - val sin = rotarySin ?: return TalkerStepResult(FloatArray(TALKER_VOCAB), FloatArray(TALKER_DIM), kCaches, vCaches) - - val inputs = LinkedHashMap() - inputs["emb"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(inputEmbed), longArrayOf(1, 1, TALKER_DIM.toLong())) - inputs["mask"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(maskData.clone()), longArrayOf(1, 1, 1, MAX_CONTEXT.toLong())) - - // cos/sin for this position: [1, 1, 128]; clamp index so we never read past - // the rotary table even if generation runs longer than the table. - val rotMax = (cos.size / TALKER_HEAD_DIM) - 1 - val rotPos = if (pos > rotMax) rotMax else pos - val cosSlice = FloatArray(TALKER_HEAD_DIM) - val sinSlice = FloatArray(TALKER_HEAD_DIM) - System.arraycopy(cos, rotPos * TALKER_HEAD_DIM, cosSlice, 0, TALKER_HEAD_DIM) - System.arraycopy(sin, rotPos * TALKER_HEAD_DIM, sinSlice, 0, TALKER_HEAD_DIM) - inputs["cos"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(cosSlice), longArrayOf(1, 1, TALKER_HEAD_DIM.toLong())) - inputs["sin"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(sinSlice), longArrayOf(1, 1, TALKER_HEAD_DIM.toLong())) - - // KV caches [1, 8, 199, 128] (NOT transposed like legacy) - val kvSize = TALKER_HEADS * KV_LEN * TALKER_HEAD_DIM // 8 * 199 * 128 - for (i in 0 until TALKER_LAYERS) { - inputs["k_${i}_in"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(kCaches[i]), - longArrayOf(1, TALKER_HEADS.toLong(), KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())) - inputs["v_${i}_in"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(vCaches[i]), - longArrayOf(1, TALKER_HEADS.toLong(), KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())) - } - - val result = session.run(inputs) - - // New format: hidden at index 0 [1,1,1024], logits at index 1 [1,1,3072] - val hidden = FloatArray(TALKER_DIM) - (result.get(0) as OnnxTensor).floatBuffer.get(hidden) - val logits = FloatArray(TALKER_VOCAB) - (result.get(1) as OnnxTensor).floatBuffer.get(logits) - - // KV caches out: [1, 8, 200, 128] — trim to [1, 8, 199, 128] (drop oldest) - val newK = Array(TALKER_LAYERS) { i -> - val full = FloatArray(TALKER_HEADS * MAX_CONTEXT * TALKER_HEAD_DIM) - (result.get(2 + i * 2) as OnnxTensor).floatBuffer.get(full) - // Drop first position: copy [1:200] → [0:199] - val trimmed = FloatArray(kvSize) - System.arraycopy(full, TALKER_HEADS * TALKER_HEAD_DIM, trimmed, 0, kvSize) - // Wait — full is [1,8,200,128] flattened. Drop pos 0 from dim 2: - // For each head h: copy full[h*200*128 + 1*128 .. h*200*128 + 200*128] → trimmed[h*199*128..] - val t = FloatArray(kvSize) - for (h in 0 until TALKER_HEADS) { - System.arraycopy(full, h * MAX_CONTEXT * TALKER_HEAD_DIM + TALKER_HEAD_DIM, - t, h * KV_LEN * TALKER_HEAD_DIM, KV_LEN * TALKER_HEAD_DIM) - } - t - } - val newV = Array(TALKER_LAYERS) { i -> - val full = FloatArray(TALKER_HEADS * MAX_CONTEXT * TALKER_HEAD_DIM) - (result.get(3 + i * 2) as OnnxTensor).floatBuffer.get(full) - val t = FloatArray(kvSize) - for (h in 0 until TALKER_HEADS) { - System.arraycopy(full, h * MAX_CONTEXT * TALKER_HEAD_DIM + TALKER_HEAD_DIM, - t, h * KV_LEN * TALKER_HEAD_DIM, KV_LEN * TALKER_HEAD_DIM) - } - t - } - - for ((_, v) in inputs) v.close() - result.close() - - return TalkerStepResult(logits, hidden, newK, newV) - } - - - /** Run code predictor — JNI .pte > TCP runner > Hexagon > CPU ONNX. */ - private var cpCallCount = 0 - private fun runCodePredictorInterleaved(pastHidden: FloatArray, cb0: Int): IntArray { - if (cpCallCount == 0) nlog("CP: pte=${cpPteModule != null}, talkerPte=${talkerPteModule != null}, et=$useEtCp, hex=$useHexagonCp, v2=$cpUsesCosSin") - cpCallCount++ - // Diagnostic override: force the ONNX Runtime CP path (runCpV2). ORT's MLAS - // kernels may have a different fp32 reduction order than ggml, potentially closer - // to PyTorch. Test hypothesis: does that reduce code divergence from Python? - if (File("/data/local/tmp/kazeia/force_cp_v2").exists() && cpUsesCosSin && cpKv != null) { - if (cpCallCount == 1) nlog("force_cp_v2: using ONNX Runtime CP (cpKv session)") - return runCpV2(pastHidden, cb0) - } - // Routing priority: - // 1. If Hexagon talker is running: MUST use hexCpForward (separate CPU process - // via socket). Running CP .pte on QNN HTP alongside Hexagon triggers err 6031 - // (documented DSP contention). - // 2. Otherwise (Talker on .pte or CPU ONNX): CP .pte JNI in the same QNN context. - // 3. Legacy fallbacks. - if (useHexagonTalker && useHexagonCp) return hexCpForward(pastHidden, cb0) - if (cpPteModule != null && (talkerPteModule != null || talkerKv != null)) return runCpPte(pastHidden, cb0) - if (useEtCp) return etCpForward(pastHidden, cb0) - if (useHexagonCp) return hexCpForward(pastHidden, cb0) - if (cpUsesCosSin && cpKv != null) return runCpV2(pastHidden, cb0) - return runCpCpu(pastHidden, cb0) - } - - /** CP via ExecuTorch .pte on NPU (QNN fp16). - * .pte inputs: emb[1,1,1024], mask[1,1,1,17], cos[1,1,128], sin[1,1,128], 10×kv[1,8,16,128] - * .pte outputs: hidden[1,1,1024], head_logits[1,15,2048], 10×kv[1,8,17,128] - */ - private fun runCpPte(pastHidden: FloatArray, cb0: Int): IntArray { - val module = cpPteModule ?: return runCpV2(pastHidden, cb0) - val cos = cpRotaryCos ?: return runCpV2(pastHidden, cb0) - val sin = cpRotarySin ?: return runCpV2(pastHidden, cb0) - - val codes = IntArray(15) - // Fixed KV caches [1, 8, 16, 128] - val kvSize = CP_KV_HEADS * CP_KV_LEN * CP_HEAD_DIM - var kCaches = Array(CP_LAYERS) { FloatArray(kvSize) } - var vCaches = Array(CP_LAYERS) { FloatArray(kvSize) } - - var emb = pastHidden - try { - for (step in 0 until 17) { - if (step == 1) { - emb = FloatArray(TALKER_DIM) - System.arraycopy(codecEmbedding!!, cb0.coerceIn(0, TALKER_VOCAB - 1) * TALKER_DIM, emb, 0, TALKER_DIM) - } else if (step >= 2) { - emb = FloatArray(TALKER_DIM) - val cembs = cpEmbeddings ?: return codes - val off = ((step - 2) * CODEBOOK_SIZE + codes[step - 2].coerceIn(0, CODEBOOK_SIZE - 1)) * TALKER_DIM - System.arraycopy(cembs, off, emb, 0, TALKER_DIM) - } - - // Build mask: last (step+1) positions active - val mask = FloatArray(CP_KV_LEN) { -1e9f } - for (p in 0..minOf(step, CP_KV_LEN - 1)) mask[CP_KV_LEN - 1 - p] = 0f - - val cosSlice = FloatArray(CP_HEAD_DIM) - System.arraycopy(cos, step * CP_HEAD_DIM, cosSlice, 0, CP_HEAD_DIM) - val sinSlice = FloatArray(CP_HEAD_DIM) - System.arraycopy(sin, step * CP_HEAD_DIM, sinSlice, 0, CP_HEAD_DIM) - - // Build EValue inputs - val inputs = arrayOf( - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(emb, longArrayOf(1, 1, TALKER_DIM.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(mask, longArrayOf(1, 1, 1, CP_KV_LEN.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(cosSlice, longArrayOf(1, 1, CP_HEAD_DIM.toLong()))), - org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(sinSlice, longArrayOf(1, 1, CP_HEAD_DIM.toLong()))), - ) - // Add KV caches - val allInputs = inputs.toMutableList() - for (i in 0 until CP_LAYERS) { - allInputs.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(kCaches[i], longArrayOf(1, CP_KV_HEADS.toLong(), CP_KV_LEN.toLong(), CP_HEAD_DIM.toLong())))) - allInputs.add(org.pytorch.executorch.EValue.from( - org.pytorch.executorch.Tensor.fromBlob(vCaches[i], longArrayOf(1, CP_KV_HEADS.toLong(), CP_KV_LEN.toLong(), CP_HEAD_DIM.toLong())))) - } - - val outputs = module.forward(*allInputs.toTypedArray()) - - // .pte outputs: hidden[1,1,1024], k0[1,8,16,128], v0[1,8,16,128], ... - val hiddenOut = outputs[0].toTensor().dataAsFloatArray - - // Head argmax using NEON SIMD (individual 8MB heads, pre-loaded) - if (step >= 1 && step - 1 < 15) { - if (cpHeadsCache == null) { - cpHeadsCache = arrayOfNulls(15) - val hp = cpHeadsPath - if (hp != null) { - for (h in 0 until 15) { - cpHeadsCache!![h] = loadNpy(hp.replace("cp_heads.npy", "head_${h}.npy")) - } - nlog("CP heads pre-loaded: 15 × ${cpHeadsCache!![0]?.size} floats") - } - } - val cbIdx = step - 1 - val headData = cpHeadsCache?.get(cbIdx) - if (headData != null) { - codes[cbIdx] = NeonOps.headArgmax(hiddenOut, headData, CODEBOOK_SIZE, TALKER_DIM) - } - } - - // Update KV caches (output is [1,8,16,128] — fixed size, already shifted) - for (i in 0 until CP_LAYERS) { - kCaches[i] = outputs[1 + i * 2].toTensor().dataAsFloatArray - vCaches[i] = outputs[2 + i * 2].toTensor().dataAsFloatArray - } - } - } catch (e: Exception) { - nlog("CP .pte error: ${e.message}, falling back to ONNX CPU") - cpPteModule = null - return runCpV2(pastHidden, cb0) - } - - return codes - } - - /** CP V2: ONNX single-step with KV-cache, 15 lm_heads, autoregressive. */ - private fun runCpV2(pastHidden: FloatArray, cb0: Int): IntArray { - val env = ortEnv ?: return IntArray(15) - val session = cpKv ?: return IntArray(15) - val cos = cpRotaryCos ?: return IntArray(15) - val sin = cpRotarySin ?: return IntArray(15) - val headPath = cpHeadsPath ?: return IntArray(15) - val cembs = cpEmbeddings ?: return IntArray(15) - - val codes = IntArray(15) - // Fixed KV: size=CP_KV_LEN always, with mask for active positions - // Dynamic KV: starts empty, grows - val kvSize = if (cpFixedKv) CP_KV_HEADS * CP_KV_LEN * CP_HEAD_DIM else 0 - var kCaches = Array(CP_LAYERS) { FloatArray(kvSize) } - var vCaches = Array(CP_LAYERS) { FloatArray(kvSize) } - val cpMask = if (cpFixedKv) FloatArray(CP_KV_LEN) { -1e9f } else null - - // Step 0: hidden state - var emb = pastHidden - for (step in 0 until 17) { - if (step == 1) { - // cb0 embedding from talker codec_embedding - emb = FloatArray(TALKER_DIM) - System.arraycopy(codecEmbedding!!, cb0.coerceIn(0, TALKER_VOCAB - 1) * TALKER_DIM, emb, 0, TALKER_DIM) - } else if (step >= 2) { - // codec_embedding from CP's own tables - emb = FloatArray(TALKER_DIM) - val off = ((step - 2) * CODEBOOK_SIZE + codes[step - 2].coerceIn(0, CODEBOOK_SIZE - 1)) * TALKER_DIM - System.arraycopy(cembs, off, emb, 0, TALKER_DIM) - } - - val kvLen = if (cpFixedKv) CP_KV_LEN else kCaches[0].size / (CP_KV_HEADS * CP_HEAD_DIM) - val cosSlice = FloatArray(CP_HEAD_DIM) - System.arraycopy(cos, step * CP_HEAD_DIM, cosSlice, 0, CP_HEAD_DIM) - val sinSlice = FloatArray(CP_HEAD_DIM) - System.arraycopy(sin, step * CP_HEAD_DIM, sinSlice, 0, CP_HEAD_DIM) - - val inputs = LinkedHashMap() - inputs["emb"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(emb), longArrayOf(1, 1, TALKER_DIM.toLong())) - if (cpFixedKv) { - // Fixed KV mask: after step N, the last (N+1) positions are valid - // The model shifts left internally, so valid positions are right-aligned - val mask = FloatArray(CP_KV_LEN) { -1e9f } - for (p in 0..step) mask[CP_KV_LEN - 1 - p] = 0f - inputs["mask"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(mask), longArrayOf(1, 1, 1, CP_KV_LEN.toLong())) - } - inputs["cos"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(cosSlice), longArrayOf(1, CP_HEAD_DIM.toLong())) - inputs["sin"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(sinSlice), longArrayOf(1, CP_HEAD_DIM.toLong())) - for (i in 0 until CP_LAYERS) { - inputs["k_${i}_in"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(kCaches[i]), - longArrayOf(1, CP_KV_HEADS.toLong(), kvLen.toLong(), CP_HEAD_DIM.toLong())) - inputs["v_${i}_in"] = OnnxTensor.createTensor(env, FloatBuffer.wrap(vCaches[i]), - longArrayOf(1, CP_KV_HEADS.toLong(), kvLen.toLong(), CP_HEAD_DIM.toLong())) - } - - val result = session.run(inputs) - val hidden = FloatArray(TALKER_DIM) - (result.get(0) as OnnxTensor).floatBuffer.get(hidden) - - // Extract code from lm_head: hidden @ head[cb]^T → argmax - if (step >= 1 && step - 1 < 15) { - val cbIdx = step - 1 - // Lazy-load head into cache (8MB each, loaded once) - if (cpHeadsCache == null) cpHeadsCache = arrayOfNulls(15) - val cache = cpHeadsCache!! - if (cache[cbIdx] == null) { - cache[cbIdx] = loadNpy(headPath.replace("cp_heads.npy", "head_${cbIdx}.npy")) - } - val headData = cache[cbIdx]!! - var best = 0; var bestVal = Float.NEGATIVE_INFINITY - for (j in 0 until CODEBOOK_SIZE) { - var dot = 0f - val off = j * TALKER_DIM - for (k in 0 until TALKER_DIM) dot += hidden[k] * headData[off + k] - if (dot > bestVal) { bestVal = dot; best = j } - } - codes[cbIdx] = best - } - - // Update KV caches - if (cpFixedKv) { - // Fixed: output is same size as input (CP_KV) - val fixedSize = CP_KV_HEADS * CP_KV_LEN * CP_HEAD_DIM - kCaches = Array(CP_LAYERS) { i -> - FloatArray(fixedSize).also { (result.get(1 + i * 2) as OnnxTensor).floatBuffer.get(it) } - } - vCaches = Array(CP_LAYERS) { i -> - FloatArray(fixedSize).also { (result.get(2 + i * 2) as OnnxTensor).floatBuffer.get(it) } - } - } else { - // Dynamic: grows by 1 - val newKvLen = kvLen + 1 - val newKvSize = CP_KV_HEADS * newKvLen * CP_HEAD_DIM - kCaches = Array(CP_LAYERS) { i -> - FloatArray(newKvSize).also { (result.get(1 + i * 2) as OnnxTensor).floatBuffer.get(it) } - } - vCaches = Array(CP_LAYERS) { i -> - FloatArray(newKvSize).also { (result.get(2 + i * 2) as OnnxTensor).floatBuffer.get(it) } - } - } - - for ((_, v) in inputs) v.close() - result.close() - } - - return codes - } - - - /** CP via ONNX CPU KV-cache: 17 single-token steps with dynamic KV. ~180ms. */ - private fun runCpCpu(pastHidden: FloatArray, cb0: Int): IntArray { - val env = ortEnv ?: return IntArray(15) - val cpModel = cpKv ?: return IntArray(15) - val cpEmbs = cpEmbeddings ?: return IntArray(15) - - val codes = IntArray(15) - // Dynamic KV caches: grow from [1,8,0,128] to [1,8,16,128] - var kCaches = Array(CP_LAYERS) { FloatArray(0) } - var vCaches = Array(CP_LAYERS) { FloatArray(0) } - - // Step 0: hidden state - var emb = pastHidden - for (step in 0 until 17) { - if (step == 1) { - emb = FloatArray(TALKER_DIM) - System.arraycopy(codecEmbedding!!, cb0.coerceIn(0, TALKER_VOCAB - 1) * TALKER_DIM, emb, 0, TALKER_DIM) - } else if (step >= 2) { - emb = FloatArray(TALKER_DIM) - val off = ((step - 2) * CODEBOOK_SIZE + codes[step - 2].coerceIn(0, CODEBOOK_SIZE - 1)) * TALKER_DIM - System.arraycopy(cpEmbs, off, emb, 0, TALKER_DIM) - } - - val totalLen = step + 1 - val inputs = LinkedHashMap() - inputs["input_embeds"] = OnnxTensor.createTensor(env, - FloatBuffer.wrap(emb), longArrayOf(1, 1, TALKER_DIM.toLong())) - inputs["attention_mask"] = OnnxTensor.createTensor(env, - FloatBuffer.wrap(FloatArray(totalLen)), longArrayOf(1, 1, 1, totalLen.toLong())) - inputs["position_ids"] = OnnxTensor.createTensor(env, - LongBuffer.wrap(longArrayOf(step.toLong())), longArrayOf(1, 1)) - for (i in 0 until CP_LAYERS) { - val kvLen = kCaches[i].size / (CP_KV_HEADS * CP_HEAD_DIM) - inputs["k_${i}_in"] = OnnxTensor.createTensor(env, - FloatBuffer.wrap(kCaches[i]), longArrayOf(1, CP_KV_HEADS.toLong(), kvLen.toLong(), CP_HEAD_DIM.toLong())) - inputs["v_${i}_in"] = OnnxTensor.createTensor(env, - FloatBuffer.wrap(vCaches[i]), longArrayOf(1, CP_KV_HEADS.toLong(), kvLen.toLong(), CP_HEAD_DIM.toLong())) - } - - val result = cpModel.run(inputs) - - // Extract head logits [1, 15, 2048] and code for this step - if (step >= 1 && step - 1 < 15) { - val headLogits = FloatArray(15 * CODEBOOK_SIZE) - (result.get(1) as OnnxTensor).floatBuffer.get(headLogits) - val cbIdx = step - 1 - val headOff = cbIdx * CODEBOOK_SIZE - var maxIdx = 0; var maxVal = Float.NEGATIVE_INFINITY - for (j in 0 until CODEBOOK_SIZE) { - if (headLogits[headOff + j] > maxVal) { maxVal = headLogits[headOff + j]; maxIdx = j } - } - codes[cbIdx] = maxIdx - } - - // Update KV caches (dynamic size grows by 1 each step) - val newKvSize = CP_KV_HEADS * totalLen * CP_HEAD_DIM - kCaches = Array(CP_LAYERS) { i -> - FloatArray(newKvSize).also { (result.get(2 + i * 2) as OnnxTensor).floatBuffer.get(it) } - } - vCaches = Array(CP_LAYERS) { i -> - FloatArray(newKvSize).also { (result.get(3 + i * 2) as OnnxTensor).floatBuffer.get(it) } - } - - for ((_, v) in inputs) v.close() - result.close() - } - - return codes - } - - /** - * Trim trailing silence/noise from audio. - * Duplicate removed — see trimTrailingSilence below. - */ - - /** Sample from logits with temperature scaling and top-K filtering */ - private fun sampleTopK(logits: FloatArray, temperature: Float = 0.9f, topK: Int = 50): Int { - // Find top-K indices - val indices = logits.indices.sortedByDescending { logits[it] }.take(topK) - - // Temperature-scaled softmax over top-K - val maxLogit = logits[indices[0]] - val expValues = FloatArray(indices.size) - var sumExp = 0f - for (i in indices.indices) { - val scaled = (logits[indices[i]] - maxLogit) / temperature - expValues[i] = kotlin.math.exp(scaled) - sumExp += expValues[i] - } - - // Categorical sampling - val r = Math.random().toFloat() * sumExp - var cumSum = 0f - for (i in indices.indices) { - cumSum += expValues[i] - if (cumSum >= r) return indices[i] - } - return indices.last() - } - - // ==================== Code Predictor ==================== - - // ==================== Speech Decoder ==================== - - /** Decode 16 codebooks to audio in chunks, with overlap for seamless concatenation */ - private fun decodeChunked(codebooks: Array, numRealTokens: Int): ShortArray { - val totalTokens = codebooks[0].size - - // Décodeur ggml C++ pur (Phase 5) : voie principale. - // codes [16][T] → wav PCM16 [T*1920], full forward sans chunking. - decoderGgml?.let { ggml -> - try { - val tFull = System.currentTimeMillis() - val pcm = ggml.decode(codebooks, numRealTokens) - if (pcm != null) { - nlog("decodeWhole (ggml C++) : ${System.currentTimeMillis() - tFull}ms pour $numRealTokens tokens → ${pcm.size} samples") - return pcm - } - nlog("ggml decode returned null → fallback .pte") - } catch (e: Exception) { - nlog("ggml decode failed (${e.message}) → fallback .pte") - } - } - - // Décodeur officiel Qwen3-TTS .pte ExecuTorch (dyn-T natif) : fallback - decoderPteModule?.let { mod -> - try { - val tFull = System.currentTimeMillis() - // Build long tensor [1, 16, numRealTokens] - val flat = LongArray(NUM_CODEBOOKS * numRealTokens) - for (cb in 0 until NUM_CODEBOOKS) { - for (t in 0 until numRealTokens) { - flat[cb * numRealTokens + t] = codebooks[cb][t].toLong() - } - } - val codesT = org.pytorch.executorch.Tensor.fromBlob( - flat, longArrayOf(1, NUM_CODEBOOKS.toLong(), numRealTokens.toLong()) - ) - val out = mod.forward(org.pytorch.executorch.EValue.from(codesT)) - val wav = out[0].toTensor().dataAsFloatArray - nlog("decodeWhole (.pte dyn-T) : ${System.currentTimeMillis()-tFull}ms pour $numRealTokens tokens → ${wav.size} samples") - return ShortArray(wav.size) { - (wav[it].coerceIn(-1f, 1f) * 32767).toInt().toShort() - } - } catch (e: Exception) { - nlog("decodeWhole .pte failed (${e.message}) → fallback ONNX") - } - } - - // ONNX dyn-T : décode TOUT le segment d'un coup, peu importe la taille. - // Plus de chunking SEQ_LEN=60, plus de crossfade, plus de transient - // BigVGAN aux frontières → fini le « beg beg beg ». - if (decoderDyntLayout) { - try { - val tFull = System.currentTimeMillis() - val quantizedFull = vqDecodeVariable(codebooks, numRealTokens) - val fullAudio = runSpeechDecoderV2Variable(quantizedFull, numRealTokens) - nlog("decodeWhole (dyn-T) : ${System.currentTimeMillis()-tFull}ms pour $numRealTokens tokens") - val trimSamples = minOf(numRealTokens * SAMPLES_PER_TOKEN, fullAudio.size) - return fullAudio.copyOf(trimSamples) - } catch (e: Exception) { - nlog("decodeWhole dyn-T failed (${e.message}) → fallback chunked legacy") - } - } - - if (totalTokens <= SEQ_LEN) { - val quantized = vqDecode(codebooks) - val fullAudio = runSpeechDecoder(quantized) - val trimSamples = minOf(numRealTokens * SAMPLES_PER_TOKEN, fullAudio.size) - return fullAudio.copyOf(trimSamples) - } - - val overlapSamples = CHUNK_OVERLAP * SAMPLES_PER_TOKEN - var result = ShortArray(0) - var pos = 0 - - while (pos < numRealTokens) { - // Stratégie align-to-end (cf. chunked_decode officiel Qwen3-TTS) : - // chaque chunk passe au BigVGAN exactement SEQ_LEN tokens RÉELS, - // jamais de padding (codebook[0] ou last_token = signal artificiel - // qui contamine la receptive field). Pour le dernier chunk, on - // recule pos pour aligner la fin sur numRealTokens. Le crossfade - // sur l'overlap (potentiellement plus large que CHUNK_OVERLAP) - // gomme le transient gauche du chunk re-aligné. - val tokensLeft = numRealTokens - pos - val readStart = if (tokensLeft < SEQ_LEN && numRealTokens >= SEQ_LEN) { - numRealTokens - SEQ_LEN - } else { - pos - } - val chunkCodes = Array(NUM_CODEBOOKS) { cb -> - IntArray(SEQ_LEN) { t -> - val srcIdx = readStart + t - if (srcIdx < numRealTokens) codebooks[cb][srcIdx] else 0 - } - } - - val quantized = vqDecode(chunkCodes) - val chunkAudio = runSpeechDecoder(quantized) - - // chunkAudio couvre les tokens [readStart..readStart+SEQ_LEN]. On - // veut garder uniquement les samples des tokens [pos..end] où - // end = min(readStart+SEQ_LEN, numRealTokens). Skip = (pos-readStart)*1920. - val skipSamples = (pos - readStart) * SAMPLES_PER_TOKEN - val endTok = minOf(readStart + SEQ_LEN, numRealTokens) - val keepTokens = endTok - pos - val keepSamples = minOf(keepTokens * SAMPLES_PER_TOKEN, chunkAudio.size - skipSamples) - val trimmed = chunkAudio.copyOfRange(skipSamples, skipSamples + keepSamples) - - if (pos == 0) { - result = trimmed - } else { - // Crossfade over the overlap region - val fadeLen = minOf(overlapSamples, result.size, trimmed.size) - for (i in 0 until fadeLen) { - val alpha = i.toFloat() / fadeLen - val mixed = ((1f - alpha) * result[result.size - fadeLen + i] + alpha * trimmed[i]).toInt() - .coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() - result[result.size - fadeLen + i] = mixed - } - // Append the non-overlapping part - if (fadeLen < trimmed.size) { - val newPart = trimmed.copyOfRange(fadeLen, trimmed.size) - val combined = ShortArray(result.size + newPart.size) - System.arraycopy(result, 0, combined, 0, result.size) - System.arraycopy(newPart, 0, combined, result.size, newPart.size) - result = combined - } - } - - pos += EFFECTIVE_CHUNK - } - return result - } - - /** VQ decode: 16 codebooks → quantized [1, 512, SEQ_LEN] */ - private fun vqDecode(codebooks: Array): FloatArray { - val firstCb = firstCodebook ?: return FloatArray(0) - val restCbs = restCodebooks ?: return FloatArray(0) - val firstProj = firstOutputProj ?: return FloatArray(0) - val restProj = restOutputProj ?: return FloatArray(0) - - val qFirst = FloatArray(CODEBOOK_DIM * SEQ_LEN) - for (t in 0 until SEQ_LEN) { - val idx = codebooks[0][t] - System.arraycopy(firstCb, idx * CODEBOOK_DIM, qFirst, t * CODEBOOK_DIM, CODEBOOK_DIM) - } - val quantized = FloatArray(HIDDEN_DIM * SEQ_LEN) - for (i in 0 until HIDDEN_DIM) { - for (t in 0 until SEQ_LEN) { - var sum = 0f - for (d in 0 until CODEBOOK_DIM) { - sum += firstProj[i * CODEBOOK_DIM + d] * qFirst[t * CODEBOOK_DIM + d] - } - quantized[i * SEQ_LEN + t] = sum - } - } - - val restSum = FloatArray(CODEBOOK_DIM * SEQ_LEN) - for (cb in 0 until 15) { - val cbData = restCbs[cb] - for (t in 0 until SEQ_LEN) { - val idx = codebooks[cb + 1][t] - for (d in 0 until CODEBOOK_DIM) { - restSum[t * CODEBOOK_DIM + d] += cbData[idx * CODEBOOK_DIM + d] - } - } - } - for (i in 0 until HIDDEN_DIM) { - for (t in 0 until SEQ_LEN) { - var sum = 0f - for (d in 0 until CODEBOOK_DIM) { - sum += restProj[i * CODEBOOK_DIM + d] * restSum[t * CODEBOOK_DIM + d] - } - quantized[i * SEQ_LEN + t] += sum - } - } - - return quantized - } - - /** Run speech decoder: quantized → pre_conv → preprocessor → ConvNet → audio */ - private fun runSpeechDecoder(quantized: FloatArray): ShortArray { - val env = ortEnv!! - if (decoderOnCpu || decoderOnGpu) { - return runSpeechDecoderV2(quantized) - } - val qTensor = OnnxTensor.createTensor( - env, FloatBuffer.wrap(quantized), longArrayOf(1, HIDDEN_DIM.toLong(), SEQ_LEN.toLong()) - ) - val pcResult = preConv!!.run(mapOf("x" to qTensor)) - val pcOut = pcResult[0] as OnnxTensor - nlog("pre_conv: ${qTensor.info.shape.contentToString()} → ${pcOut.info.shape.contentToString()}") - qTensor.close() - - val ppResult = preprocessor!!.run(mapOf("pre_conv_out" to pcOut)) - val ppOut = ppResult[0] as OnnxTensor - nlog("preprocessor: → ${ppOut.info.shape.contentToString()}") - pcResult.close() - - val cdResult = convDecoder!!.run(mapOf("hidden" to ppOut)) - val cdOut = cdResult[0] as OnnxTensor - nlog("conv_decoder: → ${cdOut.info.shape.contentToString()}") - @Suppress("UNCHECKED_CAST") - val audioFloat = (cdOut.value as Array>)[0][0] - ppResult.close() - cdResult.close() - - return ShortArray(audioFloat.size) { - (audioFloat[it].coerceIn(-1f, 1f) * 32767).toInt().toShort() - } - } - - /** Variante full-length du V2 decoder : passe `seqLen` tokens au lieu de SEQ_LEN. - * Utilisée pour décoder un segment entier d'un coup quand le décodeur tourne - * sur CPU (où il n'y a pas la contrainte fixed-size NPU). - * Si l'ONNX a été exporté sans dynamic_axes, ça crashera ici → fallback chunking. */ - private fun runSpeechDecoderV2Variable(quantized: FloatArray, seqLen: Int): ShortArray { - val env = ortEnv!! - val td0 = System.currentTimeMillis() - val qTensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(quantized), - longArrayOf(1, HIDDEN_DIM.toLong(), seqLen.toLong())) - val pcResult = preConv!!.run(mapOf(preConv!!.inputNames.first() to qTensor)) - nlog("V2-full pre_conv: ${System.currentTimeMillis()-td0}ms (seqLen=$seqLen)") - qTensor.close() - - // Avec les ONNX dyn-T natifs ([1, 1024, T]), on passe directement le pcOut - // au preprocessor sans transpose. Avec les anciens ONNX V2 (T fixed=60, - // input [1, T, 1024]), il faut transposer côté host. Le mode - // dyn-T n'est jamais combiné avec ces anciens ONNX (le block load() les - // exclut), donc on prend toujours le path native ici. - val pcOnnx = pcResult[0] as OnnxTensor - val ptResult = preprocessor!!.run(mapOf(preprocessor!!.inputNames.first() to pcOnnx)) - val ptOut = ptResult[0] as OnnxTensor - val td2 = System.currentTimeMillis() - nlog("V2-full pre_transformer: ${td2-td0}ms (cumul)") - pcResult.close() - - val cdResult = convDecoder!!.run(mapOf(convDecoder!!.inputNames.first() to ptOut)) - val cdOut = cdResult[0] as OnnxTensor - nlog("V2-full BigVGAN: ${System.currentTimeMillis()-td2}ms") - @Suppress("UNCHECKED_CAST") - val audioFloat = (cdOut.value as Array>)[0][0] - ptResult.close() - cdResult.close() - - return ShortArray(audioFloat.size) { - (audioFloat[it].coerceIn(-1f, 1f) * 32767).toInt().toShort() - } - } - - /** Variante full-length du VQ decode : pareil que vqDecode mais pour `seqLen` au lieu de SEQ_LEN. */ - private fun vqDecodeVariable(codebooks: Array, seqLen: Int): FloatArray { - val firstCb = firstCodebook ?: return FloatArray(0) - val restCbs = restCodebooks ?: return FloatArray(0) - val firstProj = firstOutputProj ?: return FloatArray(0) - val restProj = restOutputProj ?: return FloatArray(0) - - val qFirst = FloatArray(CODEBOOK_DIM * seqLen) - for (t in 0 until seqLen) { - val idx = codebooks[0][t] - System.arraycopy(firstCb, idx * CODEBOOK_DIM, qFirst, t * CODEBOOK_DIM, CODEBOOK_DIM) - } - val quantized = FloatArray(HIDDEN_DIM * seqLen) - for (i in 0 until HIDDEN_DIM) { - for (t in 0 until seqLen) { - var sum = 0f - for (d in 0 until CODEBOOK_DIM) { - sum += firstProj[i * CODEBOOK_DIM + d] * qFirst[t * CODEBOOK_DIM + d] - } - quantized[i * seqLen + t] = sum - } - } - val restSum = FloatArray(CODEBOOK_DIM * seqLen) - for (cb in 0 until 15) { - val cbData = restCbs[cb] - for (t in 0 until seqLen) { - val idx = codebooks[cb + 1][t] - for (d in 0 until CODEBOOK_DIM) { - restSum[t * CODEBOOK_DIM + d] += cbData[idx * CODEBOOK_DIM + d] - } - } - } - for (i in 0 until HIDDEN_DIM) { - for (t in 0 until seqLen) { - var sum = 0f - for (d in 0 until CODEBOOK_DIM) { - sum += restProj[i * CODEBOOK_DIM + d] * restSum[t * CODEBOOK_DIM + d] - } - quantized[i * seqLen + t] += sum - } - } - return quantized - } - - /** V2 decoder: quantized[1,512,60] → pre_conv → transpose → pre_transformer → decoder → audio */ - private fun runSpeechDecoderV2(quantized: FloatArray): ShortArray { - val env = ortEnv!! - val td0 = System.currentTimeMillis() - // pre_conv: [1,512,60] → [1,1024,60] - val qTensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(quantized), - longArrayOf(1, HIDDEN_DIM.toLong(), SEQ_LEN.toLong())) - val pcResult = preConv!!.run(mapOf(preConv!!.inputNames.first() to qTensor)) - val pcOutRaw = (pcResult[0] as OnnxTensor).floatBuffer - val pcData = FloatArray(1024 * SEQ_LEN) - pcOutRaw.get(pcData) - val td1 = System.currentTimeMillis() - nlog("V2 pre_conv: ${td1-td0}ms") - qTensor.close() - - // Transpose [1,1024,60] → [1,60,1024] - val transposed = FloatArray(SEQ_LEN * 1024) - for (c in 0 until 1024) { - for (t in 0 until SEQ_LEN) { - transposed[t * 1024 + c] = pcData[c * SEQ_LEN + t] - } - } - val ptInput = OnnxTensor.createTensor(env, FloatBuffer.wrap(transposed), - longArrayOf(1, SEQ_LEN.toLong(), 1024)) - pcResult.close() - - // pre_transformer: [1,60,1024] → [1,60,1024] - val ptResult = preprocessor!!.run(mapOf(preprocessor!!.inputNames.first() to ptInput)) - val ptOut = ptResult[0] as OnnxTensor - val td2 = System.currentTimeMillis() - nlog("V2 pre_transformer: ${td2-td1}ms") - ptInput.close() - - // decoder: [1,60,1024] → [1,1,samples] - val cdResult = convDecoder!!.run(mapOf(convDecoder!!.inputNames.first() to ptOut)) - val cdOut = cdResult[0] as OnnxTensor - nlog("V2 BigVGAN: ${System.currentTimeMillis()-td2}ms") - @Suppress("UNCHECKED_CAST") - val audioFloat = (cdOut.value as Array>)[0][0] - ptResult.close() - cdResult.close() - - return ShortArray(audioFloat.size) { - (audioFloat[it].coerceIn(-1f, 1f) * 32767).toInt().toShort() - } - } - - // ==================== Loading ==================== - - private fun loadVqCodebooks(path: String) { - firstCodebook = loadNpy("$path/vq_rvq_first_vq_layers_0_codebook.npy") - firstOutputProj = loadNpy("$path/vq_rvq_first_output_proj_w.npy") - restOutputProj = loadNpy("$path/vq_rvq_rest_output_proj_w.npy") - restCodebooks = Array(15) { i -> - loadNpy("$path/vq_rvq_rest_vq_layers_${i}_codebook.npy") - } - nlog("VQ codebooks loaded (16 × [${CODEBOOK_SIZE}, ${CODEBOOK_DIM}])") - } - - /** Load a numpy .npy file as float array */ - private fun loadNpy(path: String): FloatArray { - val file = File(path) - if (!file.exists()) { - nlog("WARN: $path not found") - return FloatArray(0) - } - val bytes = file.readBytes() - val headerLen = (bytes[8].toInt() and 0xFF) or ((bytes[9].toInt() and 0xFF) shl 8) - val dataOffset = 10 + headerLen - val numFloats = (bytes.size - dataOffset) / 4 - val result = FloatArray(numFloats) - val bb = ByteBuffer.wrap(bytes, dataOffset, bytes.size - dataOffset) - .order(ByteOrder.LITTLE_ENDIAN) - bb.asFloatBuffer().get(result) - return result - } - - // ==================== Audio Playback ==================== - - private fun playAudio(audioData: ShortArray, onComplete: () -> Unit) { - stop() - val bufferSize = audioData.size * 2 - audioTrack = AudioTrack.Builder() - .setAudioAttributes(AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_MEDIA) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build()) - .setAudioFormat(AudioFormat.Builder() - .setSampleRate(SR) - .setEncoding(AudioFormat.ENCODING_PCM_16BIT) - .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) - .build()) - .setBufferSizeInBytes(bufferSize) - .setTransferMode(AudioTrack.MODE_STATIC) - .build() - - audioTrack?.apply { - write(audioData, 0, audioData.size) - setNotificationMarkerPosition(audioData.size) - setPlaybackPositionUpdateListener(object : AudioTrack.OnPlaybackPositionUpdateListener { - override fun onMarkerReached(track: AudioTrack?) { onComplete() } - override fun onPeriodicNotification(track: AudioTrack?) {} - }) - play() - } - } - - /** - * Diagnostic path: decode Python's captured codes directly, bypassing talker+CP. - * Used to isolate whether audio degradation comes from our talker/CP predictions - * (different codes than Python) vs from BigVGAN implementation differences. - * - * File format python_codes.bin: int32 n_steps, int32 n_codebooks(=16), - * then n_steps × 16 × int32 codes. - */ - fun generateFromPythonCodes(codesPath: String): ShortArray { - nlog("Direct Python-codes path: decoding $codesPath via BigVGAN only") - val t0 = System.currentTimeMillis() - val bytes = java.io.File(codesPath).readBytes() - val bb = java.nio.ByteBuffer.wrap(bytes).order(java.nio.ByteOrder.LITTLE_ENDIAN) - val nSteps = bb.int - val nCodebooks = bb.int - nlog("Python codes: $nSteps steps × $nCodebooks codebooks") - if (nCodebooks != NUM_CODEBOOKS) { - nlog("Unexpected codebook count: $nCodebooks != $NUM_CODEBOOKS"); return ShortArray(0) - } - val padLen = maxOf(nSteps, SEQ_LEN) - val allCodebooks = Array(NUM_CODEBOOKS) { IntArray(padLen) } - for (t in 0 until nSteps) { - for (cb in 0 until NUM_CODEBOOKS) { - val v = bb.int - allCodebooks[cb][t] = v.coerceIn(0, CODEBOOK_SIZE - 1) - } - } - val audio = decodeChunked(allCodebooks, nSteps) - nlog("Python-codes decode: ${System.currentTimeMillis() - t0}ms for ${audio.size.toFloat()/SR}s audio") - try { - val wavPath = "/data/local/tmp/kazeia/kazeia_PYCODES.wav" - val fos = java.io.FileOutputStream(wavPath) - val dataLen = audio.size * 2 - val header = java.nio.ByteBuffer.allocate(44).order(java.nio.ByteOrder.LITTLE_ENDIAN) - header.put("RIFF".toByteArray()); header.putInt(36 + dataLen) - header.put("WAVE".toByteArray()); header.put("fmt ".toByteArray()) - header.putInt(16); header.putShort(1); header.putShort(1) - header.putInt(SR); header.putInt(SR * 2); header.putShort(2); header.putShort(16) - header.put("data".toByteArray()); header.putInt(dataLen) - fos.write(header.array()) - val buf = java.nio.ByteBuffer.allocate(dataLen).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (s in audio) buf.putShort(s) - fos.write(buf.array()); fos.close() - nlog("WAV saved (PyCodes): $wavPath") - } catch (e: Exception) { nlog("WAV save failed: ${e.message}") } - return audio - } - - /** - * Full pipeline from pre-computed embeddings (from Python generate() capture). - * File format: int32 n_prefill, int32 n_total, then n_total × 1024 floats. - * Runs talker ONNX → CP → VQ decode → speech decoder. - */ - fun generateFromEmbeds(embedsPath: String): ShortArray { - // Diagnostic override: if python_codes.bin exists and flag is set, decode - // Python's captured codes directly to isolate BigVGAN from talker/CP predictions. - val pyCodesPath = embedsPath.replace("full_pipeline_embeds.bin", "python_codes.bin") - if (diagFlag("force_python_codes") && java.io.File(pyCodesPath).exists()) { - nlog("force_python_codes flag + file present → diagnostic codes-direct path") - return generateFromPythonCodes(pyCodesPath) - } - if (!loaded || (!nativePipelineReady && talkerPteModule == null && !useHexagonTalker && (talkerKv == null || !talkerUsesCosSin))) { - nlog("generateFromEmbeds: no talker (native=$nativePipelineReady, pte=${talkerPteModule != null}, hex=$useHexagonTalker)") - return ShortArray(0) - } - // Priority: native C++ > Java .pte > Hexagon > ONNX CPU - if (nativePipelineReady) { - return generateFromEmbedsPte(embedsPath) - } - if (talkerPteModule != null && cpPteModule != null) { - return generateFromEmbedsPte(embedsPath) - } - if (useHexagonTalker) { - return generateFromEmbedsHexagon(embedsPath) - } - nlog("Full pipeline from: $embedsPath") - val t0 = System.currentTimeMillis() - - val bytes = File(embedsPath).readBytes() - val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) - val nPrefill = bb.int - val nTotal = bb.int - val embeds = Array(nTotal) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = bb.float } } - nlog("Loaded $nTotal embeds ($nPrefill prefill + ${nTotal - nPrefill} decode)") - - val env = ortEnv!! - val session = talkerKv!! - val cos = rotaryCos!! - val sin = rotarySin!! - - // KV caches [1, 8, KV_LEN, 128] - val kvSize = TALKER_HEADS * KV_LEN * TALKER_HEAD_DIM - var kCaches = Array(TALKER_LAYERS) { FloatArray(kvSize) } - var vCaches = Array(TALKER_LAYERS) { FloatArray(kvSize) } - val maskData = FloatArray(MAX_CONTEXT) { -1e9f } - - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - var currentCb0 = -1 - var pastHidden: FloatArray? = null - - // Prefill - val tPrefill = System.currentTimeMillis() - for (step in 0 until nPrefill) { - maskData[MAX_CONTEXT - 1 - step] = 0f - val res = runTalkerStepMRoPE(env, session, embeds[step], maskData, step, kCaches, vCaches) - kCaches = res.newK; vCaches = res.newV; pastHidden = res.hidden - - if (step == nPrefill - 1) { - val logits = res.logits - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - currentCb0 = sampleTopK(logits, 0.9f, 50) - nlog("Prefill: ${System.currentTimeMillis() - tPrefill}ms, cb0=$currentCb0") - } - } - if (currentCb0 < 0 || currentCb0 == CODEC_EOS) return ShortArray(0) - - // Voice-cloned replay: detect by large nTrailing (≥ 30 → Python-captured full - // trajectory). In that mode, feed captured decode embeds verbatim (they already - // include Python's codec_sum + text_hidden) and stop exactly at nTrailing. - val nTrailingOnnx = nTotal - nPrefill - val isVoiceClonedOnnx = nTrailingOnnx >= 30 - val maxGenOnnx = if (isVoiceClonedOnnx) nTrailingOnnx else (nTrailingOnnx * 5 + 20) - var trailingIdxOnnx = 0 - val eosE = ttsEosEmbed ?: FloatArray(TALKER_DIM) - val padE = ttsPadEmbed ?: FloatArray(TALKER_DIM) - var totalTalkerMs = 0L; var totalCpMs = 0L - val forceGreedyCb0Onnx = diagFlag("force_greedy_cb0") - nlog("ONNX voice-cloned=$isVoiceClonedOnnx, maxGen=$maxGenOnnx, greedy=$forceGreedyCb0Onnx") - for (genStep in 0 until maxGenOnnx) { - val codes = IntArray(NUM_CODEBOOKS) - codes[0] = currentCb0 - - val tCp = System.currentTimeMillis() - val cpCodes = runCodePredictorInterleaved(pastHidden!!, currentCb0) - totalCpMs += System.currentTimeMillis() - tCp - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes) - generatedCb0.add(currentCb0) - - if (genStep < 3 || genStep % 20 == 0) nlog("Step ${genStep+1}: cb0=$currentCb0 cb1=${codes[1]}") - - // Voice-cloned: feed captured embed directly. Text-only: reconstruct codec_sum + trailing. - val nextEmbed = if (isVoiceClonedOnnx) { - embeds[nPrefill + genStep] - } else { - val codecSum = FloatArray(TALKER_DIM) - addEmb(codecSum, codecEmb(codes[0])) - for (cb in 1 until NUM_CODEBOOKS) addEmb(codecSum, cpEmb(cb - 1, codes[cb])) - val textE: FloatArray = when { - trailingIdxOnnx < nTrailingOnnx -> embeds[nPrefill + trailingIdxOnnx].also { trailingIdxOnnx++ } - trailingIdxOnnx == nTrailingOnnx -> { trailingIdxOnnx++; eosE } - else -> padE - } - sumEmb(codecSum, textE) - } - val absPos = nPrefill + genStep - if (absPos < MAX_CONTEXT) maskData[MAX_CONTEXT - 1 - absPos] = 0f - - val tTalker = System.currentTimeMillis() - val res = runTalkerStepMRoPE(env, session, nextEmbed, maskData, nPrefill + genStep, kCaches, vCaches) - totalTalkerMs += System.currentTimeMillis() - tTalker - kCaches = res.newK; vCaches = res.newV; pastHidden = res.hidden - - val logits = res.logits - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - val nextCb0 = if (forceGreedyCb0Onnx) { - var b = 0; var bv = logits[0] - for (j in 1 until logits.size) if (logits[j] > bv) { bv = logits[j]; b = j } - b - } else sampleTopK(logits, 0.9f, 50) - - if (!isVoiceClonedOnnx) { - if (nextCb0 == CODEC_EOS) { nlog("EOS at step ${genStep + 2}"); break } - if (generatedCb0.size >= 9 && generatedCb0.takeLast(9).all { it == nextCb0 }) { - nlog("Degeneration at step ${genStep + 2}"); break - } - } - currentCb0 = nextCb0 - } - - val n = allCodes.size - nlog("Generated $n tokens | Talker: ${totalTalkerMs}ms (${totalTalkerMs/maxOf(n,1)}ms/step) | CP: ${totalCpMs}ms (${totalCpMs/maxOf(n,1)}ms/step)") - - // Diagnostic: dump our generated codes for comparison with Python capture - try { - val codesPath = "/data/local/tmp/kazeia/tablet_codes.bin" - val fos = java.io.FileOutputStream(codesPath) - val buf = ByteBuffer.allocate(8 + n * NUM_CODEBOOKS * 4).order(ByteOrder.LITTLE_ENDIAN) - buf.putInt(n); buf.putInt(NUM_CODEBOOKS) - for (t in 0 until n) for (cb in 0 until NUM_CODEBOOKS) buf.putInt(allCodes[t][cb]) - fos.write(buf.array()); fos.close() - nlog("Tablet codes dumped (ONNX path): $codesPath") - } catch (e: Exception) { nlog("Codes dump failed: ${e.message}") } - - if (n == 0) return ShortArray(0) - - // Decode - val padLen = maxOf(n, SEQ_LEN) - val allCodebooks = Array(NUM_CODEBOOKS) { cb -> - IntArray(padLen) { t -> if (t < n) allCodes[t][cb] else 0 } - } - val audio = decodeChunked(allCodebooks, n) - nlog("Total: ${System.currentTimeMillis() - t0}ms for ${audio.size.toFloat()/SR}s audio") - - try { - val wavPath = "/data/local/tmp/kazeia/kazeia_ONNX.wav" - val fos = java.io.FileOutputStream(wavPath) - val dataLen = audio.size * 2 - val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN) - header.put("RIFF".toByteArray()); header.putInt(36 + dataLen) - header.put("WAVE".toByteArray()); header.put("fmt ".toByteArray()) - header.putInt(16); header.putShort(1); header.putShort(1) - header.putInt(SR); header.putInt(SR * 2); header.putShort(2); header.putShort(16) - header.put("data".toByteArray()); header.putInt(dataLen) - fos.write(header.array()) - val buf = ByteBuffer.allocate(dataLen).order(ByteOrder.LITTLE_ENDIAN) - for (s in audio) buf.putShort(s) - fos.write(buf.array()); fos.close() - nlog("WAV saved (ONNX): $wavPath (${audio.size} samples)") - } catch (e: Exception) { nlog("WAV save (ONNX) failed: ${e.message}") } - - return audio - } - - /** Full pipeline using .pte JNI talker + CP on NPU from pre-computed embeddings. */ - private fun generateFromEmbedsPte(embedsPath: String): ShortArray { - nlog("Full pipeline (PTE) from: $embedsPath") - val t0 = System.currentTimeMillis() - val bytes = File(embedsPath).readBytes() - val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) - - // Detect format by comparing the file size to what each layout would predict. - // single-segment layout: 8 + nTotal*1024*4 bytes (int nPrefill, int nTotal, embeds) - // multi-segment layout: n_segments * (8 + nTotal_i*1024*4) + 4 bytes header - // For a single-segment file, `firstInt == nPrefill` and - // `8 + secondInt*1024*4 == file.length`. If that equation holds, it's single-segment; - // otherwise we treat firstInt as n_segments. - val firstInt = bb.int - val secondInt = bb.int - val fileLen = bytes.size.toLong() - val singleSegmentSize = 8L + secondInt.toLong() * TALKER_DIM * 4 - val isSingleSegment = secondInt > 0 && secondInt < 100000 && fileLen == singleSegmentSize - val isMultiSegment = !isSingleSegment && firstInt in 1..100 - bb.position(0) - bb.int // re-consume firstInt so downstream reads stay aligned - // (we'll re-read secondInt below when building the single-segment path) - - if (isMultiSegment && talkerPteModule != null && cpPteModule != null) { - return generateMultiSegment(bb, firstInt, t0) - } - - // Legacy single-segment format - val nPrefill = if (isMultiSegment) { bb.position(0); bb.int; bb.int } else firstInt - val nTotal = bb.int - val embeds = Array(nTotal) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = bb.float } } - nlog("Loaded $nTotal embeds ($nPrefill prefill + ${nTotal - nPrefill} decode)") - - val allCodes: Array - // Check if capture mode requested (force Java path to capture embeds) - val captureMode = File("/data/local/tmp/kazeia/capture_mode").exists() - // Native C++ pipeline using SAME Java Module instances (no quality loss) - if (!captureMode && talkerPteModule != null && cpPteModule != null) { - // C++ loop on Java's Module instances — same QNN compilation, no JNI overhead - val prefillFlat = FloatArray(nPrefill * TALKER_DIM) - for (i in 0 until nPrefill) System.arraycopy(embeds[i], 0, prefillFlat, i * TALKER_DIM, TALKER_DIM) - val nTrailing = nTotal - nPrefill - val trailingFlat = if (nTrailing > 0) FloatArray(nTrailing * TALKER_DIM).also { arr -> - for (i in 0 until nTrailing) System.arraycopy(embeds[nPrefill + i], 0, arr, i * TALKER_DIM, TALKER_DIM) - } else null - - // Load CP heads if not already - if (cpAllHeads == null) { - val headsFile = java.io.File("/data/local/tmp/kazeia/models/cp_heads.bin") - if (headsFile.exists()) { - val hb = headsFile.readBytes() - cpAllHeads = FloatArray(hb.size / 4) - ByteBuffer.wrap(hb).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer().get(cpAllHeads!!) - } - } - - // Ensure all data arrays are loaded - val mpath = "/data/local/tmp/kazeia/models/qwen3-tts-npu" - if (codecEmbedding == null) codecEmbedding = loadNpy("$mpath/codec_embedding.npy") - if (cpEmbeddings == null) cpEmbeddings = loadNpy("$mpath/code_predictor_embeddings.npy") - if (cpRotaryCos == null) cpRotaryCos = loadNpy("$mpath/cp_kv_v2/cp_rotary_cos.npy") - if (cpRotarySin == null) cpRotarySin = loadNpy("$mpath/cp_kv_v2/cp_rotary_sin.npy") - if (talkerPteRotaryCos == null) talkerPteRotaryCos = loadNpy("/data/local/tmp/kazeia/models/talker_pte_rotary_cos.npy") - if (talkerPteRotarySin == null) talkerPteRotarySin = loadNpy("/data/local/tmp/kazeia/models/talker_pte_rotary_sin.npy") - if (ttsEosEmbed == null || ttsPadEmbed == null) { - val mpath = "/data/local/tmp/kazeia/models/qwen3-tts-npu" - val sp = loadNpy("$mpath/tts_special_embeds.npy") - ttsBosEmbed = sp.sliceArray(0 until TALKER_DIM) - ttsEosEmbed = sp.sliceArray(TALKER_DIM until 2 * TALKER_DIM) - ttsPadEmbed = sp.sliceArray(2 * TALKER_DIM until 3 * TALKER_DIM) - } - - // Voice-cloned embeds have one entry per audio frame (already captured via - // Python's natural EOS). Heuristic: if nTrailing is large enough to be a - // per-frame capture (≥ 30), trust it as the exact length; otherwise it's a - // text-only prepare_tts_native.py file that needs the runway/boost budget. - val isVoiceCloned = nTrailing >= 30 - val maxGen = if (isVoiceCloned) nTrailing - else minOf(nTrailing * 4 + 20, 90 - nPrefill) - nlog("Running native C++ pipeline (shared Module), maxGen=$maxGen, voiceCloned=$isVoiceCloned...") - val flat = talkerPteModule!!.nativeRunTtsPipeline( - prefillFlat, nPrefill, - trailingFlat, nTrailing, - codecEmbedding ?: FloatArray(0), - cpEmbeddings ?: FloatArray(0), - cpAllHeads ?: FloatArray(0), - talkerPteRotaryCos ?: FloatArray(0), talkerPteRotarySin ?: FloatArray(0), - cpRotaryCos ?: FloatArray(0), cpRotarySin ?: FloatArray(0), - ttsEosEmbed ?: FloatArray(TALKER_DIM), ttsPadEmbed ?: FloatArray(TALKER_DIM), - maxGen - ) - if (flat == null || flat.isEmpty()) return ShortArray(0) - val nTokens = flat.size / NUM_CODEBOOKS - allCodes = Array(nTokens) { t -> IntArray(NUM_CODEBOOKS) { cb -> flat[t * NUM_CODEBOOKS + cb] } } - nlog("Native pipeline: $nTokens tokens") - } else { - // Fallback: Java pipeline - val prefillEmbeds = embeds.sliceArray(0 until nPrefill).toList() - val trailingEmbeds = if (nPrefill < nTotal) embeds.sliceArray(nPrefill until nTotal).toList() else emptyList() - allCodes = runInterleavedPteFromEmbeds(prefillEmbeds, trailingEmbeds, nTotal - nPrefill) - } - - if (allCodes.isEmpty()) return ShortArray(0) - val numRealTokens = allCodes.size - val padLen = maxOf(numRealTokens, SEQ_LEN) - val allCodebooks = Array(NUM_CODEBOOKS) { cb -> - IntArray(padLen) { t -> if (t < numRealTokens) allCodes[t][cb] else 0 } - } - - val t3 = System.currentTimeMillis() - val rawAudio = decodeChunked(allCodebooks, numRealTokens) - nlog("Decode: ${System.currentTimeMillis() - t3}ms") - - // Trim trailing noise/silence: scan from end, find last loud frame - val audio = trimTrailingSilence(rawAudio) - nlog("Trimmed: ${rawAudio.size} → ${audio.size} samples (${(rawAudio.size-audio.size)/SR.toFloat()}s removed)") - - val totalMs = System.currentTimeMillis() - t0 - val audioDur = audio.size.toFloat() / SR - nlog("Total: ${totalMs}ms for ${audioDur}s") - - // Save WAV file for validation - try { - val wavPath = "/data/local/tmp/kazeia/kazeia_PTE_NPU.wav" - val fos = java.io.FileOutputStream(wavPath) - val dataLen = audio.size * 2 - val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN) - header.put("RIFF".toByteArray()); header.putInt(36 + dataLen) - header.put("WAVE".toByteArray()); header.put("fmt ".toByteArray()) - header.putInt(16); header.putShort(1); header.putShort(1) - header.putInt(SR); header.putInt(SR * 2); header.putShort(2); header.putShort(16) - header.put("data".toByteArray()); header.putInt(dataLen) - fos.write(header.array()) - val buf = ByteBuffer.allocate(dataLen).order(ByteOrder.LITTLE_ENDIAN) - for (s in audio) buf.putShort(s) - fos.write(buf.array()); fos.close() - nlog("WAV saved: $wavPath (${audio.size} samples)") - } catch (e: Exception) { - nlog("WAV save failed: ${e.message}") - } - - return audio - } - - /** PTE pipeline from pre-computed embeddings (prefill + trailing). */ - private fun runInterleavedPteFromEmbeds( - prefillEmbeds: List, trailingEmbeds: List, maxGenTokens: Int, - // Invoked synchronously after each generated step with (stepIdx, 16-codebook codes). - // Streaming callers use it to dispatch SEQ_LEN-sized chunks to the BigVGAN pipeline - // as soon as they are ready. null preserves the original batch behaviour. - onCodeStep: ((step: Int, codes: IntArray) -> Unit)? = null - ): Array { - val talkerMod = talkerPteModule ?: return emptyArray() - val cpMod = cpPteModule ?: return emptyArray() - val tCos = talkerPteRotaryCos ?: return emptyArray() - val tSin = talkerPteRotarySin ?: return emptyArray() - val eosE = ttsEosEmbed ?: return emptyArray() - val padE = ttsPadEmbed ?: return emptyArray() - - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - - val tkvSize = TALKER_HEADS * TALKER_PTE_KV_LEN * TALKER_HEAD_DIM - var tK = Array(TALKER_LAYERS) { FloatArray(tkvSize) } - var tV = Array(TALKER_LAYERS) { FloatArray(tkvSize) } - val maskData = FloatArray(TALKER_PTE_KV_LEN) { -1e9f } - - var pos = 0; var currentCb0 = -1; var pastHidden: FloatArray? = null - var trailingIdx = 0 - - // Helper to run one talker step - fun talkerStep(emb: FloatArray): Pair { - val maskIdx = TALKER_PTE_KV_LEN - 1 - minOf(pos, TALKER_PTE_KV_LEN - 1) - if (maskIdx >= 0) maskData[maskIdx] = 0f - - val posIdx = minOf(pos, tCos.size / TALKER_HEAD_DIM - 1) - val cosSlice = FloatArray(TALKER_HEAD_DIM); System.arraycopy(tCos, posIdx * TALKER_HEAD_DIM, cosSlice, 0, TALKER_HEAD_DIM) - val sinSlice = FloatArray(TALKER_HEAD_DIM); System.arraycopy(tSin, posIdx * TALKER_HEAD_DIM, sinSlice, 0, TALKER_HEAD_DIM) - - val inputs = mutableListOf( - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(emb, longArrayOf(1, 1, TALKER_DIM.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(maskData.clone(), longArrayOf(1, 1, 1, TALKER_PTE_KV_LEN.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(cosSlice, longArrayOf(1, 1, TALKER_HEAD_DIM.toLong()))), - org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(sinSlice, longArrayOf(1, 1, TALKER_HEAD_DIM.toLong()))) - ) - for (i in 0 until TALKER_LAYERS) { - inputs.add(org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(tK[i], longArrayOf(1, TALKER_HEADS.toLong(), TALKER_PTE_KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())))) - inputs.add(org.pytorch.executorch.EValue.from(org.pytorch.executorch.Tensor.fromBlob(tV[i], longArrayOf(1, TALKER_HEADS.toLong(), TALKER_PTE_KV_LEN.toLong(), TALKER_HEAD_DIM.toLong())))) - } - - val out = talkerMod.forward(*inputs.toTypedArray()) - val hidden = out[0].toTensor().dataAsFloatArray - val logits = out[1].toTensor().dataAsFloatArray - for (i in 0 until TALKER_LAYERS) { - tK[i] = out[2 + i * 2].toTensor().dataAsFloatArray - tV[i] = out[3 + i * 2].toTensor().dataAsFloatArray - } - pos++ - return Pair(hidden, logits) - } - - // Capture embeds for C++ reuse - val capturedEmbeds = mutableListOf() - - // ===== PREFILL ===== - val tPrefill = System.currentTimeMillis() - for (step in prefillEmbeds.indices) { - capturedEmbeds.add(prefillEmbeds[step].clone()) - val (h, logits) = talkerStep(prefillEmbeds[step]) - pastHidden = h - if (step == 0 && File("/data/local/tmp/kazeia/dump_compare").exists()) { - try { - java.io.FileOutputStream("/data/local/tmp/kazeia/pte_input_step0.bin").use { fos -> - val bb = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in prefillEmbeds[0]) bb.putFloat(v); fos.write(bb.array()) - } - java.io.FileOutputStream("/data/local/tmp/kazeia/pte_hidden_step0.bin").use { fos -> - val bb = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in h) bb.putFloat(v); fos.write(bb.array()) - } - File("/data/local/tmp/kazeia/pte_input_step0.bin").setReadable(true, false) - File("/data/local/tmp/kazeia/pte_hidden_step0.bin").setReadable(true, false) - var hMin = h[0]; var hMax = h[0]; var hSum = 0.0 - for (v in h) { if (v < hMin) hMin = v; if (v > hMax) hMax = v; hSum += v } - nlog("DUMP_CMP pte step0 hidden: min=$hMin max=$hMax mean=${hSum/h.size}") - } catch (e: Exception) { nlog("dump_compare pte failed: ${e.message}") } - } - if (step == prefillEmbeds.size - 1) { - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - currentCb0 = sampleTopK(logits, 0.9f, 50) - } - } - nlog("Prefill (PTE): ${System.currentTimeMillis() - tPrefill}ms, ${prefillEmbeds.size} steps, cb0=$currentCb0") - if (currentCb0 < 0 || currentCb0 == CODEC_EOS) return emptyArray() - - // ===== GENERATION ===== - var totalTalkerMs = 0L; var totalCpMs = 0L - for (genStep in 0 until maxGenTokens) { - val codes = IntArray(NUM_CODEBOOKS); codes[0] = currentCb0 - - val tCp0 = System.currentTimeMillis() - val cpCodes = runCpPte(pastHidden!!, currentCb0) - totalCpMs += System.currentTimeMillis() - tCp0 - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes); generatedCb0.add(currentCb0) - onCodeStep?.invoke(genStep, codes) - - if (genStep < 3) nlog("Step ${genStep+1}: cb0=$currentCb0 cb1=${codes[1]}") - - // Next talker input: use pre-computed decode embed if available - val nextEmbed: FloatArray - if (trailingIdx < trailingEmbeds.size) { - nextEmbed = trailingEmbeds[trailingIdx]; trailingIdx++ - } else { - val codecSum = FloatArray(TALKER_DIM) - addEmb(codecSum, codecEmb(codes[0])) - for (cb in 1 until NUM_CODEBOOKS) addEmb(codecSum, cpEmb(cb - 1, codes[cb])) - nextEmbed = sumEmb(codecSum, padE) - } - capturedEmbeds.add(nextEmbed.clone()) - - val tTalker0 = System.currentTimeMillis() - val (h, logits) = talkerStep(nextEmbed) - totalTalkerMs += System.currentTimeMillis() - tTalker0 - pastHidden = h - - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - val nextCb0 = sampleTopK(logits, 0.9f, 50) - - if (nextCb0 == CODEC_EOS) { nlog("EOS at step ${genStep + 2}"); break } - if (generatedCb0.size >= 9 && generatedCb0.takeLast(9).all { it == nextCb0 }) { - nlog("Degeneration at step ${genStep + 2}"); break - } - currentCb0 = nextCb0 - } - - val n = allCodes.size - nlog("Generated $n tokens | Talker(PTE): ${totalTalkerMs}ms (${totalTalkerMs/maxOf(n,1)}ms/step) | CP(PTE): ${totalCpMs}ms (${totalCpMs/maxOf(n,1)}ms/step)") - - // Save captured embeds - if (capturedEmbeds.isNotEmpty()) { - try { - val capPath = "/data/local/tmp/kazeia/captured_embeds.bin" - val nPrefill = prefillEmbeds.size - val fos = java.io.FileOutputStream(capPath) - val hdr = java.nio.ByteBuffer.allocate(8).order(java.nio.ByteOrder.LITTLE_ENDIAN) - hdr.putInt(nPrefill); hdr.putInt(capturedEmbeds.size) - fos.write(hdr.array()) - for (emb in capturedEmbeds) { - val buf = java.nio.ByteBuffer.allocate(TALKER_DIM * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN) - for (v in emb) buf.putFloat(v) - fos.write(buf.array()) - } - fos.close() - nlog("Captured ${capturedEmbeds.size} embeds → $capPath (${nPrefill} prefill + ${capturedEmbeds.size - nPrefill} decode)") - } catch (e: Exception) { nlog("Capture save failed: ${e.message}") } - } - - return allCodes.toTypedArray() - } - - /** Multi-segment pipeline: process each segment independently, concatenate audio. */ - private fun generateMultiSegment(bb: ByteBuffer, nSegments: Int, t0: Long): ShortArray { - nlog("Multi-segment: $nSegments segments") - val allAudio = mutableListOf() - - // Ensure data arrays loaded - val mpath = "/data/local/tmp/kazeia/models/qwen3-tts-npu" - if (cpAllHeads == null) { - val hf = java.io.File("/data/local/tmp/kazeia/models/cp_heads.bin") - if (hf.exists()) { val hb = hf.readBytes(); cpAllHeads = FloatArray(hb.size/4); ByteBuffer.wrap(hb).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer().get(cpAllHeads!!) } - } - if (codecEmbedding == null) codecEmbedding = loadNpy("$mpath/codec_embedding.npy") - if (cpEmbeddings == null) cpEmbeddings = loadNpy("$mpath/code_predictor_embeddings.npy") - if (cpRotaryCos == null) cpRotaryCos = loadNpy("$mpath/cp_kv_v2/cp_rotary_cos.npy") - if (cpRotarySin == null) cpRotarySin = loadNpy("$mpath/cp_kv_v2/cp_rotary_sin.npy") - if (talkerPteRotaryCos == null) talkerPteRotaryCos = loadNpy("$mpath/talker_pte_rotary_cos.npy") - if (talkerPteRotarySin == null) talkerPteRotarySin = loadNpy("$mpath/talker_pte_rotary_sin.npy") - if (ttsEosEmbed == null) { val sp = loadNpy("$mpath/tts_special_embeds.npy"); ttsBosEmbed=sp.sliceArray(0 until TALKER_DIM); ttsEosEmbed=sp.sliceArray(TALKER_DIM until 2*TALKER_DIM); ttsPadEmbed=sp.sliceArray(2*TALKER_DIM until 3*TALKER_DIM) } - - for (seg in 0 until nSegments) { - val nPrefill = bb.int; val nTotal = bb.int - val embeds = Array(nTotal) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = bb.float } } - nlog("Segment ${seg+1}/$nSegments: $nPrefill prefill + ${nTotal-nPrefill} decode") - - val prefillFlat = FloatArray(nPrefill * TALKER_DIM) - for (i in 0 until nPrefill) System.arraycopy(embeds[i], 0, prefillFlat, i * TALKER_DIM, TALKER_DIM) - val nTrailing = nTotal - nPrefill - val trailingFlat = if (nTrailing > 0) FloatArray(nTrailing * TALKER_DIM).also { arr -> - for (i in 0 until nTrailing) System.arraycopy(embeds[nPrefill+i], 0, arr, i*TALKER_DIM, TALKER_DIM) - } else null - - val maxGen = minOf(nTrailing * 4 + 20, 90 - nPrefill) - val flat = talkerPteModule!!.nativeRunTtsPipeline( - prefillFlat, nPrefill, trailingFlat, nTrailing, - codecEmbedding ?: FloatArray(0), cpEmbeddings ?: FloatArray(0), cpAllHeads ?: FloatArray(0), - talkerPteRotaryCos ?: FloatArray(0), talkerPteRotarySin ?: FloatArray(0), - cpRotaryCos ?: FloatArray(0), cpRotarySin ?: FloatArray(0), - ttsEosEmbed ?: FloatArray(TALKER_DIM), ttsPadEmbed ?: FloatArray(TALKER_DIM), - maxGen - ) - if (flat == null || flat.isEmpty()) continue - val nTokens = flat.size / NUM_CODEBOOKS - val segCodes = Array(nTokens) { t -> IntArray(NUM_CODEBOOKS) { cb -> flat[t * NUM_CODEBOOKS + cb] } } - nlog(" → $nTokens tokens generated") - - val padLen = maxOf(nTokens, SEQ_LEN) - val codebooks = Array(NUM_CODEBOOKS) { cb -> IntArray(padLen) { t -> if (t < nTokens) segCodes[t][cb] else 0 } } - val segAudio = decodeChunked(codebooks, nTokens) - allAudio.add(segAudio) - nlog(" → ${segAudio.size/SR.toFloat()}s audio decoded") - } - - // The fp16 NPU drift means each segment ends with a "filler" tail (page page beg). - // We detect the boundary on the audio itself: scan windows from the end, find the - // last sustained "speech-energy" window vs the trailing low-energy filler. Use - // the running peak energy as a per-segment reference so quiet vs loud voices - // both work. - val gapMs = 120 - val gapSamples = SR * gapMs / 1000 - val gap = ShortArray(gapSamples) - - fun trimSegmentTail(audio: ShortArray): ShortArray { - if (audio.size < SR / 2) return audio - val winMs = 40 - val winSamples = SR * winMs / 1000 // 960 samples - val nWin = audio.size / winSamples - if (nWin < 6) return audio - val rms = FloatArray(nWin) - for (w in 0 until nWin) { - var s = 0.0 - val o = w * winSamples - for (i in 0 until winSamples) { val x = audio[o + i].toFloat() / 32768f; s += x * x } - rms[w] = kotlin.math.sqrt(s / winSamples).toFloat() - } - // Reference: peak in the first 70% of the segment (definitely speech) - var peak = 0f - val refEnd = (nWin * 7) / 10 - for (w in 0 until refEnd) if (rms[w] > peak) peak = rms[w] - // Walk backward from the end, find the last window where energy >= 35% of peak - // sustained over 3 consecutive windows (filler tail has lower & more variable energy). - val thr = peak * 0.35f - var lastSpeech = nWin - 1 - for (w in nWin - 1 downTo 2) { - if (rms[w] >= thr && rms[w-1] >= thr && rms[w-2] >= thr) { lastSpeech = w; break } - } - // Add a tiny fade-out window after to keep the last syllable's natural decay - val keepWin = (lastSpeech + 2).coerceAtMost(nWin - 1) - val keepSamples = (keepWin + 1) * winSamples - return audio.copyOf(keepSamples) - } - - val trimmedAudio = allAudio.map { trimSegmentTail(it) } - for ((i, seg) in trimmedAudio.withIndex()) { - nlog(" Trim seg ${i+1}: ${allAudio[i].size/SR.toFloat()}s -> ${seg.size/SR.toFloat()}s") - } - - val totalSamples = trimmedAudio.sumOf { it.size } + (trimmedAudio.size - 1) * gapSamples - val result = ShortArray(totalSamples) - var offset = 0 - for ((i, seg) in trimmedAudio.withIndex()) { - System.arraycopy(seg, 0, result, offset, seg.size); offset += seg.size - if (i < trimmedAudio.size - 1) { System.arraycopy(gap, 0, result, offset, gapSamples); offset += gapSamples } - } - - val totalMs = System.currentTimeMillis() - t0 - nlog("Total: ${totalMs}ms for ${totalSamples/SR.toFloat()}s ($nSegments segments)") - - // Save WAV - try { - val wavPath = "/data/local/tmp/kazeia/kazeia_PTE_NPU.wav" - val fos = java.io.FileOutputStream(wavPath) - val dataLen = result.size * 2 - val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN) - header.put("RIFF".toByteArray()); header.putInt(36 + dataLen) - header.put("WAVE".toByteArray()); header.put("fmt ".toByteArray()) - header.putInt(16); header.putShort(1); header.putShort(1) - header.putInt(SR); header.putInt(SR * 2); header.putShort(2); header.putShort(16) - header.put("data".toByteArray()); header.putInt(dataLen) - fos.write(header.array()) - val buf = ByteBuffer.allocate(dataLen).order(ByteOrder.LITTLE_ENDIAN) - for (s in result) buf.putShort(s) - fos.write(buf.array()); fos.close() - nlog("WAV saved: $wavPath ($totalSamples samples)") - } catch (e: Exception) { nlog("WAV save failed: ${e.message}") } - - return result - } - - /** No-op trim — garbage post-text has high energy, can't distinguish from speech. - * Length is controlled by maxTokens = trailing count instead. */ - private fun trimTrailingSilence(audio: ShortArray): ShortArray = audio - - /** Full pipeline using Hexagon talker + Hexagon CP from pre-computed embeddings. */ - private fun generateFromEmbedsHexagon(embedsPath: String): ShortArray { - nlog("Full pipeline (Hexagon) from: $embedsPath") - val t0 = System.currentTimeMillis() - - val bytes = File(embedsPath).readBytes() - val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) - val nPrefill = bb.int; val nTotal = bb.int - val embeds = Array(nTotal) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = bb.float } } - nlog("Loaded $nTotal embeds ($nPrefill prefill + ${nTotal - nPrefill} decode)") - - hexReset() - - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - var totalTalkerMs = 0L; var totalCpMs = 0L - - // Prefill - val tPrefill = System.currentTimeMillis() - val prefillResults = hexForward(embeds.take(nPrefill)) - nlog("Prefill (Hex): ${System.currentTimeMillis() - tPrefill}ms, ${prefillResults.size} steps") - - if (prefillResults.isEmpty()) return ShortArray(0) - var pastHidden = prefillResults.last().first - val prefillLogits = prefillResults.last().second - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) prefillLogits[j] = Float.NEGATIVE_INFINITY } - - // Diagnostic-only cb0 sampling controls (DEBUG builds only). - // Empirical finding (thesis-relevant): temperature and greedy vs stochastic - // give similar perceptual audio quality because the cross-architecture logits - // drift (x86 AVX ↔ ARM64 HMX fp16) is the dominant error source, not sampling. - // Production always uses temp=0.9 stochastic matching Python's default. - val cb0Temp: Float = diagFile("cb0_temp")?.let { - try { it.readText().trim().toFloat() } catch (e: Exception) { 0.9f } - } ?: 0.9f - if (cb0Temp != 0.9f) nlog("cb0_temp override: $cb0Temp") - - val forceGreedyCb0 = diagFlag("force_greedy_cb0") - if (forceGreedyCb0) nlog("force_greedy_cb0: using argmax for cb0 sampling (incl. prefill)") - - // Apply greedy/temperature to PREFILL sample too — previously this was always - // sampleTopK(0.9) which made step 0 (the "B" of "Bonjour") stochastic even when - // greedy was requested. This is the most audible source of tremor on the very - // first syllable. - var currentCb0 = if (forceGreedyCb0) { - var b = 0; var bv = prefillLogits[0] - for (j in 1 until prefillLogits.size) if (prefillLogits[j] > bv) { bv = prefillLogits[j]; b = j } - b - } else sampleTopK(prefillLogits, cb0Temp, 50) - nlog("Prefill cb0=$currentCb0 (greedy=$forceGreedyCb0)") - - // Voice-cloned single-file replay: nTotal-nPrefill captured decode embeds that - // already contain codec_sum_python + text_hidden at each step. Feed them directly - // to the Hexagon talker (no re-sum from NPU codes). Stops at maxGen = nTrailing - // because Python already decided where to stop. Hexagon's dynamic KV means no - // eviction of the prefill context — this is the whole point of the test. - val nTrailingHex = nTotal - nPrefill - val isVoiceCloned = nTrailingHex >= 30 - nlog("Hex voice-cloned mode: $isVoiceCloned ($nTrailingHex decode steps)") - - // Diagnostic (DEBUG only): inject Python-captured cb0 at each step. This is - // how we proved tablet CP is 99.76% bit-identical to PyTorch when cb0 matches - // — the tremor is entirely caused by divergent cb0 trajectories, not CP drift. - val injectPyCb0 = diagFlag("force_inject_pycb0") - val pyCodesFile = File("/data/local/tmp/kazeia/models/qwen3-tts-npu/python_codes.bin") - var pyCb0: IntArray? = null - if (injectPyCb0 && pyCodesFile.exists()) { - val pb = ByteBuffer.wrap(pyCodesFile.readBytes()).order(ByteOrder.LITTLE_ENDIAN) - val n = pb.int; val cb = pb.int - pyCb0 = IntArray(n) { - val v = pb.int // cb0 - for (j in 1 until cb) pb.int // skip cb1..cb(cb-1) - v - } - nlog("injectPyCb0: loaded ${pyCb0!!.size} cb0 values (cb=$cb) from python_codes.bin") - } else if (injectPyCb0) { - nlog("injectPyCb0: flag set but python_codes.bin not found at ${pyCodesFile.absolutePath}") - } - - // Seed the very first cb0 with Python's value too if injecting — otherwise step 0's - // cb0 comes from tablet prefill sampling and doesn't match. - if (injectPyCb0 && pyCb0 != null && pyCb0!!.isNotEmpty()) { - currentCb0 = pyCb0!![0] - nlog("injectPyCb0: overriding step 0 cb0 -> ${pyCb0!![0]}") - } - - for (genStep in 0 until nTrailingHex) { - val codes = IntArray(NUM_CODEBOOKS); codes[0] = currentCb0 - val tCp = System.currentTimeMillis() - val cpCodes = runCodePredictorInterleaved(pastHidden, currentCb0) - totalCpMs += System.currentTimeMillis() - tCp - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes); generatedCb0.add(currentCb0) - - if (genStep < 3 || genStep % 20 == 0) nlog("Step ${genStep+1}: cb0=$currentCb0 cb1=${codes[1]}") - - // Feed captured decode embed verbatim (already includes Python's codec_sum + text) - val nextEmbed = embeds[nPrefill + genStep] - - val tT = System.currentTimeMillis() - val results = hexForward(listOf(nextEmbed)) - totalTalkerMs += System.currentTimeMillis() - tT - if (results.isEmpty()) { nlog("Hex forward returned empty at step ${genStep+1}"); break } - pastHidden = results[0].first - val logits = results[0].second - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - - // Determine next cb0. If injecting, snap to Python's captured cb0 for step+1. - // This isolates pure CP numerical divergence: tablet gets identical embeds AND - // identical cb0 at every step, so any cb1..cb15 divergence must come from CP - // numerics (ONNX Runtime vs PyTorch) alone. - val nextIdx = genStep + 1 - currentCb0 = if (injectPyCb0 && pyCb0 != null && nextIdx < pyCb0!!.size) { - pyCb0!![nextIdx] - } else if (forceGreedyCb0) { - var b = 0; var bv = logits[0] - for (j in 1 until logits.size) if (logits[j] > bv) { bv = logits[j]; b = j } - b - } else sampleTopK(logits, cb0Temp, 50) - } - - val n = allCodes.size - nlog("Generated $n tokens | Talker(HEX): ${totalTalkerMs}ms (${totalTalkerMs/maxOf(n,1)}ms/step) | CP: ${totalCpMs}ms (${totalCpMs/maxOf(n,1)}ms/step)") - - // Stop hexagon runners before decode ONLY if decoder uses HTP (DSP conflict) - if (!decoderOnCpu && !decoderOnGpu) { - hexStopRunner() - } - - if (n == 0) return ShortArray(0) - val padLen = maxOf(n, SEQ_LEN) - val allCodebooks = Array(NUM_CODEBOOKS) { cb -> IntArray(padLen) { t -> if (t < n) allCodes[t][cb] else 0 } } - - // Diagnostic: dump our generated codes for comparison with Python capture - try { - val codesPath = "/data/local/tmp/kazeia/tablet_codes.bin" - val fos = java.io.FileOutputStream(codesPath) - val buf = ByteBuffer.allocate(8 + n * NUM_CODEBOOKS * 4).order(ByteOrder.LITTLE_ENDIAN) - buf.putInt(n); buf.putInt(NUM_CODEBOOKS) - for (t in 0 until n) for (cb in 0 until NUM_CODEBOOKS) buf.putInt(allCodes[t][cb]) - fos.write(buf.array()); fos.close() - nlog("Tablet codes dumped: $codesPath ($n steps × $NUM_CODEBOOKS)") - } catch (e: Exception) { nlog("Codes dump failed: ${e.message}") } - - val audio = decodeChunked(allCodebooks, n) - nlog("Total: ${System.currentTimeMillis() - t0}ms for ${audio.size.toFloat()/SR}s") - - // Save WAV for inspection - try { - val wavPath = "/data/local/tmp/kazeia/kazeia_HEX.wav" - val fos = java.io.FileOutputStream(wavPath) - val dataLen = audio.size * 2 - val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN) - header.put("RIFF".toByteArray()); header.putInt(36 + dataLen) - header.put("WAVE".toByteArray()); header.put("fmt ".toByteArray()) - header.putInt(16); header.putShort(1); header.putShort(1) - header.putInt(SR); header.putInt(SR * 2); header.putShort(2); header.putShort(16) - header.put("data".toByteArray()); header.putInt(dataLen) - fos.write(header.array()) - val buf = ByteBuffer.allocate(dataLen).order(ByteOrder.LITTLE_ENDIAN) - for (s in audio) buf.putShort(s) - fos.write(buf.array()); fos.close() - nlog("WAV saved (Hex): $wavPath (${audio.size} samples)") - } catch (e: Exception) { nlog("WAV save (Hex) failed: ${e.message}") } - - return audio - } - - /** - * Run the Hexagon talker + CP generation loop on a single segment's embeds - * and return the decoded audio. Extracted from generateFromEmbedsHexagon so - * both single-segment playback and the streaming multi-segment path share - * exactly the same generation path (no quality drift between modes). - * - * Caller is responsible for hexReset() before first call of a request. - * Subsequent calls (segments 2..N in multi-segment mode) must hexReset() - * between segments so the talker KV-cache doesn't carry stale context. - */ - private fun runHexSegmentFromEmbeds( - prefillEmbeds: List, - trailingEmbeds: List, - segIdx: Int = 0 - ): ShortArray { - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - var totalTalkerMs = 0L; var totalCpMs = 0L - - // Prefill - val tPrefill = System.currentTimeMillis() - val prefillResults = hexForward(prefillEmbeds) - nlog("Seg ${segIdx+1} prefill: ${System.currentTimeMillis() - tPrefill}ms, ${prefillResults.size} steps") - if (prefillResults.isEmpty()) return ShortArray(0) - - var pastHidden = prefillResults.last().first - val prefillLogits = prefillResults.last().second - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) prefillLogits[j] = Float.NEGATIVE_INFINITY } - var currentCb0 = sampleTopK(prefillLogits, 0.9f, 50) - - val nTrailing = trailingEmbeds.size - for (genStep in 0 until nTrailing) { - val codes = IntArray(NUM_CODEBOOKS); codes[0] = currentCb0 - val tCp = System.currentTimeMillis() - val cpCodes = runCodePredictorInterleaved(pastHidden, currentCb0) - totalCpMs += System.currentTimeMillis() - tCp - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes); generatedCb0.add(currentCb0) - - val nextEmbed = trailingEmbeds[genStep] - val tT = System.currentTimeMillis() - val results = hexForward(listOf(nextEmbed)) - totalTalkerMs += System.currentTimeMillis() - tT - if (results.isEmpty()) { nlog("Seg ${segIdx+1}: hex empty at step ${genStep+1}"); break } - pastHidden = results[0].first - val logits = results[0].second - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - currentCb0 = sampleTopK(logits, 0.9f, 50) - } - - val n = allCodes.size - nlog("Seg ${segIdx+1} generated $n tokens | Talker(HEX): ${totalTalkerMs}ms | CP: ${totalCpMs}ms") - if (n == 0) return ShortArray(0) - - // cb0 can hit CODEC_EOS (> CODEBOOK_SIZE) on longer phrases — the original - // single-segment Hexagon path never exercises this because the short Baer - // probe stayed well inside the decode budget. Clamp any out-of-vocab code - // to 0 (silence) so vqDecode can't read past the codebook buffer. - val padLen = maxOf(n, SEQ_LEN) - val allCodebooks = Array(NUM_CODEBOOKS) { cb -> - IntArray(padLen) { t -> - if (t < n) { val v = allCodes[t][cb]; if (v in 0 until CODEBOOK_SIZE) v else 0 } else 0 - } - } - return decodeChunked(allCodebooks, n) - } - - /** - * Streaming multi-segment variant of generateFromEmbeds. Reads a multi-segment - * embeds file, generates each segment via Hexagon talker + CP sequentially, - * and invokes `onSegmentReady(idx, audio)` the moment each segment's audio is - * decoded. The callback writes to an AudioTrack in the calling coroutine so - * playback begins as soon as segment 1 finishes (~5s for a 5s segment, - * instead of ~15s for the full phrase). - * - * Each segment's raw audio is saved to /data/local/tmp/kazeia/kazeia_stream_segN.wav - * and the final concatenated audio to /data/local/tmp/kazeia/kazeia_stream_full.wav - * so the caller can inspect individual segments for quality regressions. - * - * Single-segment files are supported as a degenerate case (nSegments=1) so - * the caller doesn't need to branch on format. - */ - fun generateFromEmbedsHexagonStreaming( - embedsPath: String, - onSegmentReady: ((segIdx: Int, audio: ShortArray) -> Unit)? = null - ): ShortArray { - if (!loaded || !useHexagonTalker) { - nlog("Streaming: Hexagon talker not ready") - return ShortArray(0) - } - val t0 = System.currentTimeMillis() - val bytes = File(embedsPath).readBytes() - val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) - - // Format detection (same logic as generateFromEmbedsPte): - // single: - // multi : [ ] × nSegments - val firstInt = bb.int - val secondInt = bb.int - val fileLen = bytes.size.toLong() - val singleSize = 8L + secondInt.toLong() * TALKER_DIM * 4 - val isSingle = secondInt > 0 && secondInt < 100000 && fileLen == singleSize - val nSegments = if (isSingle) 1 else firstInt - bb.position(if (isSingle) 0 else 4) - nlog("Streaming: $nSegments segment(s), ${bytes.size} bytes") - - // Ensure a fresh runner connection for this request. Between requests - // the KV-cache carries stale state from the previous generation and - // prefill logits come out as garbage on segment 1. - hexReset() - - val segmentAudios = mutableListOf() - val gapSamples = SR * 120 / 1000 - val gap = ShortArray(gapSamples) - - for (seg in 0 until nSegments) { - // Between segments the talker KV-cache must be reset so segment 2's - // prefill logits don't contain segment 1's state. Skipping this - // produces garbled speech from segment 2 onwards. - if (seg > 0) hexReset() - - val nPrefill = bb.int - val nTotal = bb.int - val prefill = List(nPrefill) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = bb.float } } - val nTrailing = nTotal - nPrefill - val trailing = List(nTrailing) { FloatArray(TALKER_DIM).also { arr -> for (j in 0 until TALKER_DIM) arr[j] = bb.float } } - nlog("Streaming seg ${seg+1}/$nSegments: $nPrefill prefill + $nTrailing decode") - - val tSeg = System.currentTimeMillis() - val audio = runHexSegmentFromEmbeds(prefill, trailing, seg) - val segMs = System.currentTimeMillis() - tSeg - nlog("Streaming seg ${seg+1}/$nSegments: ${audio.size/SR.toFloat()}s audio in ${segMs}ms") - - // Emit to caller immediately so playback starts now. WAV dump is - // synchronous here; for true zero-lag streaming the caller can - // write to AudioTrack on its own dispatcher from the callback. - segmentAudios.add(audio) - saveWav("/data/local/tmp/kazeia/kazeia_stream_seg${seg+1}.wav", audio) - onSegmentReady?.invoke(seg, audio) - } - - // Concatenate with short gaps between segments for the full-file WAV. - // Playback path already inserted perceptual spacing via the callback order. - val total = segmentAudios.sumOf { it.size } + maxOf(0, segmentAudios.size - 1) * gapSamples - val concat = ShortArray(total) - var off = 0 - for ((i, s) in segmentAudios.withIndex()) { - System.arraycopy(s, 0, concat, off, s.size); off += s.size - if (i < segmentAudios.size - 1) { System.arraycopy(gap, 0, concat, off, gapSamples); off += gapSamples } - } - saveWav("/data/local/tmp/kazeia/kazeia_stream_full.wav", concat) - nlog("Streaming total: ${System.currentTimeMillis() - t0}ms for ${concat.size/SR.toFloat()}s ($nSegments seg)") - return concat - } - - /** - * Look up a single token in the fp16 full-vocab text-embedding table and - * return it as fp32. Uses direct ByteBuffer arithmetic so we don't - * allocate a new buffer per token — for a typical 50-token sentence the - * inner loop runs 50 × 1024 fp16→fp32 conversions. - */ - private fun textEmbFromFull(tokenId: Int): FloatArray { - val buf = textEmbedsFullBuf ?: error("Stage 2 full embeddings not loaded") - val clamped = tokenId.coerceIn(0, textEmbedsFullLen - 1) - val base = clamped * TALKER_DIM * 2 - val out = FloatArray(TALKER_DIM) - synchronized(buf) { - // MappedByteBuffer has mutable position; guard in case two - // coroutines ever race on tokenizer output concurrently. - buf.position(base) - for (j in 0 until TALKER_DIM) { - val bits = buf.short - out[j] = halfToFloat(bits) - } - } - return out - } - - /** IEEE 754 fp16 -> fp32 conversion. Handles subnormals, inf and NaN - * exactly the way Python's `torch.float16.to(float32)` does. */ - private fun halfToFloat(h: Short): Float { - val bits = h.toInt() and 0xffff - val sign = (bits ushr 15) and 0x1 - val exp = (bits ushr 10) and 0x1f - val mant = bits and 0x3ff - val f32: Int = when { - exp == 0 && mant == 0 -> sign shl 31 - exp == 0 -> { - // Subnormal: normalize by shifting until leading 1 appears. - var e = -14; var m = mant - while ((m and 0x400) == 0) { m = m shl 1; e -= 1 } - val e32 = e + 127 - (sign shl 31) or (e32 shl 23) or ((m and 0x3ff) shl 13) - } - exp == 0x1f -> (sign shl 31) or (0xff shl 23) or (mant shl 13) // inf / NaN - else -> (sign shl 31) or ((exp - 15 + 127) shl 23) or (mant shl 13) - } - return Float.fromBits(f32) - } - - // Stage 3 streaming session state. A session opens a single AudioTrack - // and a background worker that pulls sentences from a Channel and - // generates+plays them sequentially. Audio for sentence N plays while - // sentence N+1's codes are being generated on Hexagon+CP, giving a - // smoother LLM-to-voice UX than "generate all, play all". - private var sessionTrack: AudioTrack? = null - private var sessionChannel: kotlinx.coroutines.channels.Channel? = null - private var sessionJob: kotlinx.coroutines.Job? = null - private var sessionKeepAliveJob: kotlinx.coroutines.Job? = null - private var sessionFocusRequest: android.media.AudioFocusRequest? = null - // Total PCM frames queued to sessionTrack across all segments in this session. - // endStreamingSession() polls track.playbackHeadPosition until it reaches this - // count before calling stop(), so the tail sentence isn't clipped. - // Uses AtomicLong because both the session worker and the keep-alive watchdog - // call writeAndCount concurrently. - private val sessionFramesWritten = java.util.concurrent.atomic.AtomicLong(0) - // True while a real-audio generate call is in progress. The keep-alive - // watchdog skips silence injection while this is set, so silence never - // interleaves with speech inside a segment. - private val sessionGenActive = java.util.concurrent.atomic.AtomicBoolean(false) - - /** - * Open a streaming TTS session backed by a persistent AudioTrack. After - * this returns, callers feed sentences one by one via enqueueSentence(); - * each sentence is voice-cloned and its audio is written to the shared - * track as soon as it's decoded. Call endStreamingSession() to flush - * the queue and release the track. - */ - // MediaPlayer-based fallback session state. If ColorOS mutes our - // AudioTrack (as observed repeatedly — `event:muted updated source: - // clientVolume` right after play()), we instead render each segment - // as a WAV file on shared storage and play it back via MediaPlayer, - // which uses a completely different internal audio pipeline that - // doesn't get silenced by the background playback policy. - private var sessionMpQueue: kotlinx.coroutines.channels.Channel? = null - private var sessionMpJob: kotlinx.coroutines.Job? = null - private val sessionMpSegIdx = java.util.concurrent.atomic.AtomicInteger(0) - - /** - * Fires the moment a synthesized segment starts playing through the - * speaker. Carries the sentence text, audio duration, per-window RMS - * envelope (for orb amplitude) and per-window log-spaced band - * spectrogram (for the spectrum-in-sphere visualizer). All three - * share the same time axis — one entry per [ENVELOPE_WINDOW_MS]. - */ - var onSegmentPlaying: (( - sentence: String, - durationMs: Long, - rmsEnvelope: FloatArray, - spectrogram: Array - ) -> Unit)? = null - - // Debug session dump: when /data/local/tmp/kazeia/dump_tts exists, - // accumulate the full audio of the current streaming session and - // save it as one concatenated WAV at session end. Used to extract - // gold-standard tablet TTS output for chantier ggml-cpu validation - // (compare audio-A/B vs Python host references). - private val sessionDumpBuf = mutableListOf() - private var sessionDumpEnabled = false - private var sessionDumpStart = 0L - - private fun startStreamingSessionMp() { - if (sessionMpQueue != null) return - sessionMpSegIdx.set(0) - sessionDumpEnabled = File("/data/local/tmp/kazeia/dump_tts").exists() - if (sessionDumpEnabled) { - sessionDumpBuf.clear() - sessionDumpStart = System.currentTimeMillis() - val dumpDir = File("/data/local/tmp/kazeia/tts_dump") - dumpDir.mkdirs() - // Make dir world-readable so adb shell can pull the WAVs - // (default mkdirs creates 0700 owned by the app user). - try { - dumpDir.setReadable(true, false) - dumpDir.setExecutable(true, false) - dumpDir.setWritable(true, false) - } catch (_: Exception) {} - nlog("dump_tts ON — full session WAV will be saved") - } - val sentenceChan = kotlinx.coroutines.channels.Channel( - capacity = kotlinx.coroutines.channels.Channel.UNLIMITED - ) - // Pipeline: synth worker produces WAV paths, playback worker runs - // them through a pair of MediaPlayer instances chained via - // setNextMediaPlayer() so there's zero-gap transition between - // chunks (no DAC/output routing "pop" the user was hearing as - // "beg beg" with one player-per-seg). - // - // When USE_STREAMING_DECODE is on, each sentence is split into - // ~1 s WAV chunks streamed into wavChan as soon as the crossfader - // emits them. MediaPlayer's setNextMediaPlayer() chain turns the - // chunk list back into one seamless audio stream per sentence — - // and because chunk 1 can start playing while chunks 2..N are - // still being synthesized, time-to-first-audio drops roughly - // from (wall time of full sentence) to (wall time of ~20 talker/CP - // steps + 1 BigVGAN pass). - val wavChan = kotlinx.coroutines.channels.Channel( - capacity = kotlinx.coroutines.channels.Channel.UNLIMITED) - val scope = kotlinx.coroutines.CoroutineScope(kotlinx.coroutines.Dispatchers.IO) - val synthJob = scope.launch { - for (sentence in sentenceChan) { - try { - val segIdx = sessionMpSegIdx.getAndIncrement() - val tSynth = System.currentTimeMillis() - val cacheDir = context?.cacheDir?.absolutePath ?: "/data/local/tmp/kazeia" - - // Streaming path : run the talker/CP/BigVGAN pipeline with - // a callback that receives each stable audio slice from - // the crossfader. We accumulate those slices up to - // ~CHUNK_MS_TARGET of audio then emit one WAV, so we get - // fewer MediaPlayer chaining hops while still publishing - // the first sub-second of audio early. - // - // Note 2026-05-03 : streaming GGML désactivé. La voie validée - // utilise `decodeWhole` (sans chunking) car le decoder ggml - // a une receptive field convolutionnelle plus grande que - // les overlaps testés (5 tokens) → artefacts "page beg beg" - // aux bornes de chunk. Pour réduire la latence : Option A - // (recompiler libllama contre chraac-llama) qui accélère - // Talker+CP sans toucher au décodeur. - val canStreamPte = talkerPteModule != null && cpPteModule != null - if (USE_STREAMING_DECODE && canStreamPte) { - val chunkTargetSamples = SR * CHUNK_MS_TARGET / 1000 // ~24k samples - val buffer = ArrayList(chunkTargetSamples + SR) - var subIdx = 0 - var firstChunkLogged = false - - suspend fun flushChunk(isFinal: Boolean) { - if (buffer.isEmpty()) return - val arr = ShortArray(buffer.size) - for (i in arr.indices) arr[i] = buffer[i] - buffer.clear() - val wavPath = "$cacheDir/tts_seg_${segIdx}_${subIdx}.wav" - saveWav(wavPath, arr) - // Debug dump : accumulate full session audio for adb pull - if (sessionDumpEnabled) { - synchronized(sessionDumpBuf) { - for (s in arr) sessionDumpBuf.add(s) - } - } - val durationMs = arr.size * 1000L / SR - // Envelope + spectrogram are only used by the UI - // visualizer on the FIRST chunk of a segment (as - // the "this sentence just started" signal). - // Avoid the FFT cost on subsequent chunks where - // playback is already running. - val envelope = if (subIdx == 0) computeRmsEnvelope(arr) else FloatArray(0) - val spectrogram = if (subIdx == 0) computeSpectrogram(arr) else emptyArray() - val sentenceLabel = if (subIdx == 0) sentence else "" - val tag = if (subIdx == 0) "seg $segIdx chunk 0 (first)" else "seg $segIdx chunk $subIdx" - if (!firstChunkLogged) { - nlog("MP $tag synth+write = ${System.currentTimeMillis() - tSynth}ms (${durationMs}ms audio) — streaming") - firstChunkLogged = true - } else { - nlog("MP $tag (${durationMs}ms audio${if (isFinal) ", final" else ""})") - } - wavChan.send(SegmentReady(segIdx, wavPath, sentenceLabel, durationMs, envelope, spectrogram)) - subIdx++ - } - - val onPcm: (ShortArray) -> Unit = { pcm -> - // onAudio is invoked from a background IO coroutine - // inside the streaming function — safe to do - // blocking file I/O here but keep per-call minimal. - for (s in pcm) buffer.add(s) - if (buffer.size >= chunkTargetSamples) { - // Flush synchronously. runBlocking is OK since - // the consumer side (playChainedMediaPlayers) - // is on a different coroutine. - kotlinx.coroutines.runBlocking { flushChunk(false) } - } - } - generateSegmentAudioVCStreaming(sentence, segIdx, onPcm) - // Tail chunk (≤ CHUNK_MS_TARGET) after the generator - // finishes emitting and the crossfader's flushFinal - // has fired. - flushChunk(true) - nlog("MP seg $segIdx total synth = ${System.currentTimeMillis() - tSynth}ms, ${subIdx} WAV chunks emitted") - } else { - // Legacy path : one WAV per segment, synthesized fully - // before playback starts. Kept for the non-PTE fallback. - val audio = generateSegmentAudioVC(sentence, segIdx) - if (audio.isEmpty()) continue - val wavPath = "$cacheDir/tts_seg_${segIdx}.wav" - saveWav(wavPath, audio) - if (sessionDumpEnabled) { - try { - val dumpPath = "/data/local/tmp/kazeia/tts_dump/seg_${segIdx}_${sessionDumpStart}.wav" - saveWav(dumpPath, audio) - File(dumpPath).setReadable(true, false) - nlog("CPU dump : seg $segIdx → $dumpPath (${audio.size} samples)") - } catch (e: Exception) { nlog("CPU dump failed: ${e.message}") } - } - val durationMs = audio.size * 1000L / SR - val envelope = computeRmsEnvelope(audio) - val spectrogram = computeSpectrogram(audio) - nlog("MP seg $segIdx synthesized (${System.currentTimeMillis() - tSynth}ms, ${durationMs}ms audio, ${envelope.size} env × ${SPECTRUM_BANDS} bands), queued for playback") - wavChan.send(SegmentReady(segIdx, wavPath, sentence, durationMs, envelope, spectrogram)) - } - } catch (e: Exception) { - nlog("MP synth error: ${e.message}") - } - } - wavChan.close() - } - val playJob = scope.launch { playChainedMediaPlayers(wavChan) } - val combined = scope.launch { synthJob.join(); playJob.join() } - sessionMpQueue = sentenceChan; sessionMpJob = combined - nlog("streaming session opened (MediaPlayer fallback, chunked-streaming)") - } - - /** - * Drive the WAV playback pipeline with two MediaPlayer instances - * chained via setNextMediaPlayer() so each segment flows into the - * next without re-arming the audio output (which caused audible - * "pops" between segments when one player stopped and another - * started). Consumes (segIdx, wavPath) pairs from [wavChan] and - * deletes each file after it finishes playing. Suspends until the - * channel closes AND the final segment finishes. - */ - private suspend fun playChainedMediaPlayers( - wavChan: kotlinx.coroutines.channels.ReceiveChannel - ) { - val attrs = android.media.AudioAttributes.Builder() - .setUsage(android.media.AudioAttributes.USAGE_MEDIA) - .setContentType(android.media.AudioAttributes.CONTENT_TYPE_SPEECH) - .build() - - // Synchronously prepare a MediaPlayer on the current coroutine. - // Throws on failure; caller handles cleanup. - suspend fun prepareMp(path: String, segIdx: Int): android.media.MediaPlayer { - val mp = android.media.MediaPlayer() - mp.setAudioAttributes(attrs) - mp.setDataSource(path) - kotlinx.coroutines.suspendCancellableCoroutine { cont -> - mp.setOnPreparedListener { if (cont.isActive) cont.resume(Unit) {} } - mp.setOnErrorListener { _, what, extra -> - nlog("MP seg $segIdx prepare error: what=$what extra=$extra") - if (cont.isActive) cont.resume(Unit) {} - true - } - cont.invokeOnCancellation { try { mp.release() } catch (_: Exception) {} } - mp.prepareAsync() - } - return mp - } - - // Per-player book-keeping. `done` completes the moment the - // MediaPlayer's OnCompletionListener fires, so the loop can - // tell *before* calling setNextMediaPlayer whether the chain - // will actually trigger (setNextMediaPlayer on a player already - // in the Completed state is a silent no-op — that was the root - // cause of missing audio on seg 1 when synthesis ran longer - // than seg 0's playback). - class Live( - val mp: android.media.MediaPlayer, - val info: SegmentReady, - val done: kotlinx.coroutines.CompletableDeferred - ) - - fun arm(info: SegmentReady, mp: android.media.MediaPlayer): Live { - val done = kotlinx.coroutines.CompletableDeferred() - mp.setOnCompletionListener { - try { it.release() } catch (_: Exception) {} - if (!done.isCompleted) done.complete(Unit) - } - mp.setOnErrorListener { _, what, extra -> - nlog("MP seg ${info.segIdx} play error: what=$what extra=$extra") - if (!done.isCompleted) done.complete(Unit) - true - } - return Live(mp, info, done) - } - - var current: Live? = null - - try { - // Bootstrap with the first segment. - val first = wavChan.receiveCatching().getOrNull() ?: return - val firstMp = prepareMp(first.wavPath, first.segIdx) - firstMp.start() - current = arm(first, firstMp) - try { onSegmentPlaying?.invoke(first.sentence, first.durationMs, first.rmsEnvelope, first.spectrogram) } catch (_: Exception) {} - nlog("MP seg ${first.segIdx} started (${first.durationMs}ms)") - - while (true) { - val upcoming = wavChan.receiveCatching().getOrNull() ?: break - val nextMp = prepareMp(upcoming.wavPath, upcoming.segIdx) - - // Try to chain so Android auto-starts next when current - // finishes — gives zero-gap playback without re-arming - // the DAC. Skipped if current has already completed - // (setNext on Completed is a no-op); we fall back to an - // explicit start() below in that case. - var chained = false - try { - if (!current!!.done.isCompleted) { - current!!.mp.setNextMediaPlayer(nextMp) - chained = true - } - } catch (e: Exception) { - nlog("MP seg ${upcoming.segIdx} setNext failed: ${e.message}") - } - - // Wait for current playback to finish before rotating. - current!!.done.await() - try { java.io.File(current!!.info.wavPath).delete() } catch (_: Exception) {} - - // If we never chained (or the chain raced with the - // current's completion), start next manually. Safe to - // start() again even if Android already auto-started. - val autoStarted = try { chained && (nextMp.isPlaying || nextMp.currentPosition > 0) } catch (_: Exception) { false } - if (!autoStarted) { - try { nextMp.start() } catch (e: Exception) { - nlog("MP seg ${upcoming.segIdx} manual start failed: ${e.message}") - } - nlog("MP seg ${upcoming.segIdx} started manually (chain missed)") - } else { - nlog("MP seg ${upcoming.segIdx} auto-chained") - } - - current = arm(upcoming, nextMp) - try { onSegmentPlaying?.invoke(upcoming.sentence, upcoming.durationMs, upcoming.rmsEnvelope, upcoming.spectrogram) } catch (_: Exception) {} - } - - // Drain: wait for the last player to finish. - current?.done?.await() - current?.let { try { java.io.File(it.info.wavPath).delete() } catch (_: Exception) {} } - } catch (e: Exception) { - nlog("MP playback chain error: ${e.message}") - } finally { - try { current?.mp?.release() } catch (_: Exception) {} - } - } - - /** Payload handed from the synth worker to the playback worker so - * the UI can be notified with matching text + duration when each - * segment starts playing. The [rmsEnvelope] is an optional sidecar - * array of per-ENVELOPE_WINDOW_MS RMS values normalized to [0, 1] - * that drives the audio-reactive orb visualizer without having to - * read PCM back from MediaPlayer. */ - private data class SegmentReady( - val segIdx: Int, - val wavPath: String, - val sentence: String, - val durationMs: Long, - val rmsEnvelope: FloatArray, - val spectrogram: Array - ) - - /** Compute a per-ENVELOPE_WINDOW_MS normalized RMS envelope from a - * mono 16-bit PCM buffer at [SR]. Cheap (one pass, trivially fast - * on the ~100 k samples we generate per segment) and called only - * once per segment right after synthesis. */ - private fun computeRmsEnvelope(audio: ShortArray): FloatArray { - if (audio.isEmpty()) return FloatArray(0) - val windowSamples = SR * ENVELOPE_WINDOW_MS / 1000 - val nWindows = (audio.size + windowSamples - 1) / windowSamples - val env = FloatArray(nWindows) - for (w in 0 until nWindows) { - val start = w * windowSamples - val end = minOf(start + windowSamples, audio.size) - var sumSq = 0.0 - for (i in start until end) { - val s = audio[i].toDouble() - sumSq += s * s - } - val rms = kotlin.math.sqrt(sumSq / (end - start)) - // Normalize: 32767 is full-scale; squash the upper range - // with a sqrt curve so even quiet speech shows visible - // motion without saturating on loud peaks. - env[w] = kotlin.math.sqrt((rms / 32767.0).coerceIn(0.0, 1.0)).toFloat() - } - return env - } - - /** Compute a per-window log-spaced band spectrogram used by the - * spectrum-in-sphere visualizer. Time axis aligned with the RMS - * envelope (one column per ENVELOPE_WINDOW_MS). FFT size is 1024 - * samples (~43 ms at 24 kHz), windowed with Hann and centered on - * each hop. [SPECTRUM_BANDS] log-spaced bands from 120 Hz to - * 4 kHz — covers the vocal formant range without wasting visual - * space on silent sub-100 Hz or frictive >4 kHz content. */ - private fun computeSpectrogram(audio: ShortArray): Array { - if (audio.isEmpty()) return emptyArray() - val fftSize = FFT_SIZE - val hopSamples = SR * ENVELOPE_WINDOW_MS / 1000 - val nFrames = (audio.size + hopSamples - 1) / hopSamples - // Pre-compute band edges as FFT bin indices. - val binHzRes = SR.toDouble() / fftSize - val fMin = 120.0; val fMax = 4000.0 - val bandEdges = IntArray(SPECTRUM_BANDS + 1) { i -> - val f = fMin * Math.pow(fMax / fMin, i.toDouble() / SPECTRUM_BANDS) - (f / binHzRes).toInt().coerceIn(1, fftSize / 2 - 1) - } - // Hann window — reduces spectral leakage, gives cleaner bars. - val hann = FloatArray(fftSize) { i -> - (0.5 - 0.5 * Math.cos(2.0 * Math.PI * i / (fftSize - 1))).toFloat() - } - val re = FloatArray(fftSize) - val im = FloatArray(fftSize) - val result = Array(nFrames) { FloatArray(SPECTRUM_BANDS) } - for (f in 0 until nFrames) { - // Center the window on the hop midpoint. - val center = f * hopSamples + hopSamples / 2 - val start = center - fftSize / 2 - for (i in 0 until fftSize) { - val idx = start + i - val sample = if (idx in audio.indices) audio[idx].toFloat() / 32768f else 0f - re[i] = sample * hann[i] - im[i] = 0f - } - fftInPlace(re, im) - for (b in 0 until SPECTRUM_BANDS) { - val bStart = bandEdges[b] - val bEnd = bandEdges[b + 1].coerceAtLeast(bStart + 1) - var sum = 0.0 - for (k in bStart until bEnd) { - val reK = re[k].toDouble(); val imK = im[k].toDouble() - sum += reK * reK + imK * imK - } - val mag = Math.sqrt(sum / (bEnd - bStart)) - // Log-compress + normalize. Speech energy per band rarely - // exceeds ~0.1 before log; the constants below bring the - // typical range to [0.2, 0.95] for visible bar motion. - result[f][b] = (Math.log10(1.0 + mag * 80) / Math.log10(7.0)) - .toFloat().coerceIn(0f, 1f) - } - } - return result - } - - /** In-place radix-2 Cooley–Tukey FFT. Size must be a power of 2. */ - private fun fftInPlace(re: FloatArray, im: FloatArray) { - val n = re.size - // Bit-reversal permutation. - var j = 0 - for (i in 1 until n) { - var bit = n shr 1 - while (j and bit != 0) { j = j xor bit; bit = bit shr 1 } - j = j or bit - if (i < j) { - val tr = re[i]; re[i] = re[j]; re[j] = tr - val ti = im[i]; im[i] = im[j]; im[j] = ti - } - } - // Butterflies. - var size = 2 - while (size <= n) { - val half = size / 2 - val step = n / size - val angleBase = -2.0 * Math.PI / size - var m = 0 - while (m < n) { - var k = 0 - for (i in m until m + half) { - val angle = (angleBase * k).toFloat() - val c = kotlin.math.cos(angle) - val s = kotlin.math.sin(angle) - val tRe = re[i + half] * c - im[i + half] * s - val tIm = re[i + half] * s + im[i + half] * c - re[i + half] = re[i] - tRe - im[i + half] = im[i] - tIm - re[i] = re[i] + tRe - im[i] = im[i] + tIm - k += step - } - m += size - } - size *= 2 - } - } - - private suspend fun endStreamingSessionMp() { - val chan = sessionMpQueue ?: return - chan.close() - try { sessionMpJob?.join() } catch (_: Exception) {} - sessionMpQueue = null; sessionMpJob = null - onSegmentPlaying = null - // Save concatenated full-session WAV if dump flag was set - if (sessionDumpEnabled) { - val frames = synchronized(sessionDumpBuf) { sessionDumpBuf.toShortArray() } - sessionDumpBuf.clear() - val dumpPath = "/data/local/tmp/kazeia/tts_dump/session_${sessionDumpStart}.wav" - try { - saveWav(dumpPath, frames) - // World-readable for adb pull (saveWav creates 0600 by default) - try { File(dumpPath).setReadable(true, false) } catch (_: Exception) {} - val durMs = frames.size * 1000L / SR - nlog("dump_tts saved: $dumpPath (${frames.size} samples, ${durMs}ms)") - } catch (e: Exception) { - nlog("dump_tts save failed: ${e.message}") - } - sessionDumpEnabled = false - } - nlog("streaming session closed (MediaPlayer fallback)") - } - - /** - * Play a WAV file via Android MediaPlayer and block the calling - * coroutine until playback completes. MediaPlayer uses a separate - * audio pipeline from AudioTrack so it bypasses ColorOS's AudioTrack - * hardening/muting behaviour. - */ - private suspend fun playWavBlocking(path: String, segIdx: Int) { - val t0 = System.currentTimeMillis() - suspendCancellableCoroutine { cont -> - val mp = android.media.MediaPlayer() - try { - mp.setAudioAttributes(android.media.AudioAttributes.Builder() - .setUsage(android.media.AudioAttributes.USAGE_MEDIA) - .setContentType(android.media.AudioAttributes.CONTENT_TYPE_SPEECH) - .build()) - mp.setDataSource(path) - mp.setOnPreparedListener { - nlog("MP seg $segIdx prepared, starting (prep ${System.currentTimeMillis() - t0}ms)") - it.start() - } - mp.setOnCompletionListener { - nlog("MP seg $segIdx done (${System.currentTimeMillis() - t0}ms total)") - try { it.release() } catch (_: Exception) {} - if (cont.isActive) cont.resume(Unit) {} - } - mp.setOnErrorListener { player, what, extra -> - nlog("MP seg $segIdx error: what=$what extra=$extra") - try { player.release() } catch (_: Exception) {} - if (cont.isActive) cont.resume(Unit) {} - true - } - mp.prepareAsync() - cont.invokeOnCancellation { try { mp.release() } catch (_: Exception) {} } - } catch (e: Exception) { - nlog("MP seg $segIdx setup failed: ${e.message}") - try { mp.release() } catch (_: Exception) {} - if (cont.isActive) cont.resume(Unit) {} - } - } - } - - fun startStreamingSession() { - if (USE_MEDIAPLAYER_FALLBACK) { startStreamingSessionMp(); return } - if (sessionTrack != null) return // already open - // USAGE_VOICE_COMMUNICATION routes to STREAM_VOICE_CALL, which - // ColorOS's "Audio Hardening" policy does NOT silently mute (the - // policy targets STREAM_MUSIC to preserve battery on inactive media - // apps; STREAM_VOICE_CALL is reserved for VoIP and always plays). - // Previous attempts with USAGE_MEDIA and USAGE_ASSISTANT both got - // `event:muted updated source:clientVolume` ~0.6–1 s after play() - // even with audio focus + mediaPlayback FGS, so moving off of - // STREAM_MUSIC is the only route that unblocks audible playback. - val attrs = AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build() - val track = AudioTrack.Builder() - .setAudioAttributes(attrs) - .setAudioFormat(AudioFormat.Builder() - .setEncoding(AudioFormat.ENCODING_PCM_16BIT) - .setSampleRate(SR) - .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) - .build()) - .setBufferSizeInBytes(SR * 30 * 2) // 30 s buffer; AudioTrack - // paces writes when full. - .setTransferMode(AudioTrack.MODE_STREAM) - .build() - // Request audio focus for the duration of the session. Without this - // ColorOS's Audio Hardening treats the track as background noise - // and mutes it, regardless of FGS status. We don't care about - // focus loss callbacks — if another app grabs focus mid-sentence - // that's fine, the track just gets ducked. - val am = context?.getSystemService(android.content.Context.AUDIO_SERVICE) as? android.media.AudioManager - val focusReq = android.media.AudioFocusRequest.Builder(android.media.AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) - .setAudioAttributes(attrs) - .setOnAudioFocusChangeListener { _ -> } - .build() - val focusRes = am?.requestAudioFocus(focusReq) - nlog("audio focus request: $focusRes (1=granted, 0=failed, 2=delayed)") - sessionFocusRequest = focusReq - // ColorOS mutes AudioTrack clientVolume ~1s after creation (seen in - // dumpsys audio as `event:muted updated source:clientVolume`). Force - // track volume back to 1.0 repeatedly to override. This is also - // done in the keep-alive watchdog loop below for ongoing override. - try { track.setVolume(1.0f) } catch (_: Exception) {} - track.play() - sessionFramesWritten.set(0) - sessionGenActive.set(false) - // writeAndCount is the single path through which PCM reaches the - // AudioTrack for this session, so sessionFramesWritten always stays - // in sync with what's been queued to playback hardware. AudioTrack.write - // is thread-safe, so this can be called concurrently from the session - // worker (real audio) and the keep-alive watchdog (silence padding). - val writeAndCount: (ShortArray) -> Unit = { pcm -> - if (pcm.isNotEmpty()) { - val n = track.write(pcm, 0, pcm.size) - if (n > 0) sessionFramesWritten.addAndGet(n.toLong()) - } - } - // Bootstrap silence: queue 500 ms immediately after play() so - // AudioFlinger has samples to mix from the very first cycle. - // Without this, there's a ~100 ms window between play() and the - // first watchdog tick where the track has no data and AudioFlinger - // flags it for removal. Once that happens, playbackHead sticks at - // 0 and subsequent writes go to a dead track. - val bootstrapSilence = ShortArray(SR / 2) // 500 ms - writeAndCount(bootstrapSilence) - // Keep-alive watchdog. AudioFlinger on OnePlus/ColorOS kills a track - // that underruns for ~1 s (confirmed via `prepareTracks_l BUFFER - // TIMEOUT: remove track … due to underrun on thread 29`). Our - // per-segment synthesis takes 3–5 s, which always exceeds that - // window between writes, so the track was getting silenced after - // the first ~1 s of audio played. The watchdog pads with 200 ms of - // silence any time the buffered-ahead audio drops below 400 ms, - // regardless of segment state — silence only advances playback head - // in the gaps between real audio and is never inserted inside a - // contiguous burst of real writes (those bring buffered above 400 ms - // and keep the watchdog quiet). - val keepAliveBuffer = ShortArray(SR / 5) // 200 ms of silence - val keepAliveJob = kotlinx.coroutines.CoroutineScope( - kotlinx.coroutines.Dispatchers.IO - ).launch { - var tick = 0 - while (kotlinx.coroutines.currentCoroutineContext()[kotlinx.coroutines.Job]?.isActive != false) { - kotlinx.coroutines.delay(100) - val head = track.playbackHeadPosition.toLong() and 0xFFFFFFFFL - val written = sessionFramesWritten.get() and 0xFFFFFFFFL - val buffered = written - head - val needsPad = buffered < SR * 2 / 5 // < 400 ms - if ((tick and 0x1F) == 0) { - nlog("keepAlive tick=$tick head=$head written=$written buffered=$buffered pad=$needsPad state=${track.playState}") - } - tick++ - // Override any clientVolume mute that ColorOS keeps applying. - try { track.setVolume(1.0f) } catch (_: Exception) {} - if (needsPad) writeAndCount(keepAliveBuffer) - } - } - val chan = kotlinx.coroutines.channels.Channel( - capacity = kotlinx.coroutines.channels.Channel.UNLIMITED - ) - val job = kotlinx.coroutines.CoroutineScope( - kotlinx.coroutines.Dispatchers.IO - ).launch { - var segIdx = 0 - for (sentence in chan) { - try { - sessionGenActive.set(true) - // Streaming GGML désactivé 2026-05-03 (artefacts "page beg beg" - // au boundaries de chunks — receptive field decoder > overlap). - // Voie production = decodeWhole, propre. Latence à attaquer - // via Option A (chraac-llama libllama pour Talker+CP). - if (USE_STREAMING_DECODE && talkerPteModule != null && cpPteModule != null) { - generateSegmentAudioVCStreaming(sentence, segIdx, writeAndCount) - } else { - val audio = generateSegmentAudioVC(sentence, segIdx) - writeAndCount(audio) - } - segIdx++ - } catch (e: Exception) { - nlog("session seg $segIdx error: ${e.message}") - } finally { - sessionGenActive.set(false) - } - } - } - sessionTrack = track; sessionChannel = chan; sessionJob = job - sessionKeepAliveJob = keepAliveJob - nlog("streaming session opened") - } - - /** - * Enqueue a sentence for background generation and playback in the - * session opened by startStreamingSession(). Non-blocking: returns - * immediately. Sentences play in the order they were enqueued. - */ - fun enqueueSentence(sentence: String) { - if (USE_MEDIAPLAYER_FALLBACK) { - val chan = sessionMpQueue ?: run { nlog("enqueueSentence: no MP session"); return } - val r = chan.trySend(sentence) - if (r.isFailure) nlog("enqueueSentence: MP channel full / closed") - return - } - val chan = sessionChannel ?: run { nlog("enqueueSentence: no session open"); return } - val r = chan.trySend(sentence) - if (r.isFailure) nlog("enqueueSentence: channel full / closed") - } - - /** - * Close the sentence queue, wait for all pending audio to finish - * generating (playback may continue briefly after as the AudioTrack - * drains), then release the shared track. Safe to call more than once. - */ - suspend fun endStreamingSession() { - if (USE_MEDIAPLAYER_FALLBACK) { endStreamingSessionMp(); return } - val chan = sessionChannel ?: return - chan.close() - try { sessionJob?.join() } catch (_: Exception) {} - // Stop the keep-alive watchdog BEFORE draining so it doesn't pad more - // silence onto the tail while we're waiting for the existing buffer - // to play out. - try { sessionKeepAliveJob?.cancel() } catch (_: Exception) {} - try { sessionKeepAliveJob?.join() } catch (_: Exception) {} - try { - sessionTrack?.let { track -> - // AudioTrack.stop() in MODE_STREAM DISCARDS unplayed buffered - // samples — it doesn't block for drain. Poll getPlaybackHead - // Position() until it reaches what we wrote, then stop. The - // head is a 32-bit wrap-around counter, so compare modulo. - // Cap the drain wait so a stalled track can't block us forever. - val targetFrames = sessionFramesWritten.get() - val startMs = System.currentTimeMillis() - val maxDrainMs = (targetFrames * 1000L / SR) + 500L // audio dur + 500ms slack - while (true) { - val head = track.playbackHeadPosition.toLong() and 0xFFFFFFFFL - val reached = head >= (targetFrames and 0xFFFFFFFFL) - val state = track.playState - if (reached || state != AudioTrack.PLAYSTATE_PLAYING) break - if (System.currentTimeMillis() - startMs > maxDrainMs) { - nlog("endStreamingSession: drain timeout at head=$head/$targetFrames") - break - } - kotlinx.coroutines.delay(20) - } - track.stop(); track.release() - } - } catch (_: Exception) {} - // Release audio focus after the track is fully drained and stopped. - try { - val am = context?.getSystemService(android.content.Context.AUDIO_SERVICE) as? android.media.AudioManager - sessionFocusRequest?.let { am?.abandonAudioFocusRequest(it) } - } catch (_: Exception) {} - sessionFocusRequest = null - sessionTrack = null; sessionChannel = null; sessionJob = null; sessionKeepAliveJob = null - nlog("streaming session closed") - } - - /** - * Voice-clone a single sentence and return its decoded audio. Mirrors - * the per-segment body of synthesizeTextStreaming but without WAV save - * side effects, since the streaming session only cares about PCM going - * to the AudioTrack. - */ - private fun generateSegmentAudioVC(segText: String, segIdx: Int): ShortArray { - if (bpeTokenizer == null || textEmbedsFullBuf == null || damienVoicePrefix == null || damienVoiceSuffix == null) { - nlog("generateSegmentAudioVC: Stage 2 assets missing"); return ShortArray(0) - } - val prefix = damienVoicePrefix!! - val suffix = damienVoiceSuffix!! - val codecPadEmb = codecEmb(CODEC_PAD) - val ids = bpeTokenizer!!.encode(segText) - nlog("session seg $segIdx '${segText.take(60)}' → ${ids.size} tokens") - - val prefill = ArrayList(prefix.size + ids.size + suffix.size) - for (e in prefix) prefill.add(e) - for (id in ids) prefill.add(sumEmb(textEmbFromFull(id), codecPadEmb)) - for (e in suffix) prefill.add(e) - - // Ratios observés sur ggml-cpu (Qwen3-TTS Damien) : - // - phrases courtes < 5 tok text → ~6-8 tok audio par tok text - // - phrases moyennes 10-20 tok → ~4-5 tok audio par tok text - // - phrases longues > 25 tok → ~3-4 tok audio par tok text - // On vise un peu généreux pour ne pas couper en plein mot ; le boost EOS - // termine de toute façon plus tôt si le talker l'émet naturellement. - val expectedSteps = ids.size * 4 - val cpuCap = 220 // n_ctx 256 - prefill ~36 = 220 - val maxGen = if (useCpuPath) { - minOf(expectedSteps * 2 + 20, cpuCap) - } else { - minOf(expectedSteps * 3 / 2 + 10, MAX_CONTEXT - 15) - } - // Floor : interdire le boost avant 2/3 de expectedSteps. - // Pour ids.size=15 → expectedSteps=60 → minStep=40. - // Sous ce floor, le talker doit pouvoir parler sans être interrompu. - val eosBoostMinStep = (expectedSteps * 2) / 3 - nlog("generateSegmentAudioVC: ids=${ids.size}, expectedSteps=$expectedSteps, maxGen=$maxGen (useCpuPath=$useCpuPath)") - - // Backend dispatch: ggml-cpu (llama.cpp dynamic KV) takes priority when - // active; otherwise Hexagon socket; otherwise .pte fixed-KV path. - val codes: Array = if (useCpuPath) { - runInterleavedCpuFromEmbeds(prefill, emptyList(), maxGen, eosBoostMinStep = eosBoostMinStep) - } else if (talkerSocket != null) { - hexReset() - runHexGenWithPrefill(prefill, maxGen, eosBoostMinStep) - } else if (talkerPteModule != null && cpPteModule != null) { - runInterleavedPteFromEmbeds(prefill, emptyList(), maxGen) - } else { - nlog("generateSegmentAudioVC: no talker backend available"); return ShortArray(0) - } - if (codes.isEmpty()) return ShortArray(0) - - val n = codes.size - val padLen = maxOf(n, SEQ_LEN) - // Pad au-delà des tokens réels avec le DERNIER token réel (continuation - // naturelle) plutôt que 0 (qui = codebook[0] = un vrai phonème - // contaminant la receptive field future du BigVGAN — voir decodeChunked). - val lastIdx = (n - 1).coerceAtLeast(0) - val codebooks = Array(NUM_CODEBOOKS) { cb -> - IntArray(padLen) { t -> - val src = if (t < n) t else lastIdx - val v = codes[src][cb] - if (v in 0 until CODEBOOK_SIZE) v else 0 - } - } - // 40 ms fade-out so the EOS-clipped tail decays naturally before - // the AudioTrack write. - return fadeOut(decodeChunked(codebooks, n), 40) - } - - // ---------- Streaming decode (CP ↔ BigVGAN overlap) ---------- - - /** Carrier from the talker/CP producer to the BigVGAN consumer. */ - private class ChunkMsg(val codebooks: Array, val realTokens: Int) - - /** - * Streaming variant of decodeChunked. Mirrors its semantics exactly: the - * internal `result` buffer accumulates and crossfades chunks the same - * way, so the final assembled audio is bit-identical. The difference is - * that whenever a portion of `result` becomes "stable" (no future chunk - * can modify it, i.e. anything before the last `overlapSamples`), it is - * emitted via `onAudio` immediately. `flushFinal()` emits the remaining - * tail with fadeOut applied, matching the original behaviour. - */ - private inner class StreamingCrossfader( - private val onAudio: (ShortArray) -> Unit, - // Overlap exprimé en tokens audio. Default = CHUNK_OVERLAP du chemin - // SEQ_LEN=60 (V2 ONNX, fixe). Le chemin ggml C++ utilise un overlap - // plus petit (= CHUNK_OVERLAP_GGML) car SEQ_LEN_GGML=20 → 5/4 du SEQ_LEN. - overlapTokens: Int = CHUNK_OVERLAP - ) { - private val overlapSamples = overlapTokens * SAMPLES_PER_TOKEN - private var result = ShortArray(0) - private var emittedLen = 0 - private var isFirst = true - - fun feedChunk(chunkAudio: ShortArray, realTokens: Int) { - val trimLen = minOf(realTokens * SAMPLES_PER_TOKEN, chunkAudio.size) - val trimmed = if (trimLen < chunkAudio.size) chunkAudio.copyOf(trimLen) else chunkAudio - - if (isFirst) { - result = trimmed.copyOf() - isFirst = false - } else { - val fadeLen = minOf(overlapSamples, result.size, trimmed.size) - for (i in 0 until fadeLen) { - val alpha = i.toFloat() / fadeLen - val mixed = ((1f - alpha) * result[result.size - fadeLen + i] + alpha * trimmed[i]).toInt() - .coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() - result[result.size - fadeLen + i] = mixed - } - if (fadeLen < trimmed.size) { - val newPart = trimmed.copyOfRange(fadeLen, trimmed.size) - val combined = ShortArray(result.size + newPart.size) - System.arraycopy(result, 0, combined, 0, result.size) - System.arraycopy(newPart, 0, combined, result.size, newPart.size) - result = combined - } - } - - // Hold back the last `overlapSamples` so the next chunk's - // crossfade can still mutate them; emit everything before that. - val stableEnd = (result.size - overlapSamples).coerceAtLeast(emittedLen) - if (stableEnd > emittedLen) { - val slice = result.copyOfRange(emittedLen, stableEnd) - onAudio(slice) - emittedLen = stableEnd - } - } - - /** Emit any remaining buffered samples with the trailing fadeOut. */ - fun flushFinal() { - if (emittedLen < result.size) { - val tail = result.copyOfRange(emittedLen, result.size) - onAudio(fadeOut(tail, 40)) - emittedLen = result.size - } - } - } - - /** - * Streaming variant pour le décodeur ggml C++ (`Qwen3TtsGgmlDecoder`). - * - * Différences clés vs `generateSegmentAudioVCStreaming` (V2 ONNX, fixe SEQ_LEN=60) : - * - Chunks plus petits (SEQ_LEN_GGML=20) car le décodeur ggml accepte T variable - * → bénéfice streaming sur les phrases courtes (15-40 tokens) qui sinon - * tombaient sur trailing-only et étaient identiques au mode non-streaming. - * - Overlap CHUNK_OVERLAP_GGML=5 tokens (au lieu de 10). - * - Décodeur ggml C++ direct (pas de vqDecode → quantized → V2 ONNX). API : - * `decoderGgml.decode(codebooks: Array, T: Int) → ShortArray(T*1920)` - * - * Pour 33 tokens (réponse "Je t'écoute, tout va bien.") : - * - decodeWhole : ~3.8 s avant 1er audio - * - streaming chunked : 1er chunk de 20 tokens prêt en ~2 s, 2è chunk en ~1 s, - * total compute ~3.5 s mais 1er audio audible après ~2 s - * → -1.5 à 2 s de latence perçue. - */ - private suspend fun generateSegmentAudioVCStreamingGgml( - segText: String, segIdx: Int, onAudio: (ShortArray) -> Unit - ) { - val ggml = decoderGgml ?: run { - nlog("generateSegmentAudioVCStreamingGgml: decoder ggml absent"); return - } - if (bpeTokenizer == null || textEmbedsFullBuf == null || damienVoicePrefix == null || damienVoiceSuffix == null) { - nlog("generateSegmentAudioVCStreamingGgml: Stage 2 assets missing"); return - } - if (!useCpuPath) { - nlog("generateSegmentAudioVCStreamingGgml: useCpuPath=false (Talker+CP indisponibles)"); return - } - val SEQ_LEN_GGML = 20 - val CHUNK_OVERLAP_GGML = 5 - val EFFECTIVE_CHUNK_GGML = SEQ_LEN_GGML - CHUNK_OVERLAP_GGML // 15 - - val prefix = damienVoicePrefix!! - val suffix = damienVoiceSuffix!! - val codecPadEmb = codecEmb(CODEC_PAD) - val ids = bpeTokenizer!!.encode(segText) - nlog("session seg $segIdx (ggml-stream) '${segText.take(60)}' → ${ids.size} tokens, chunk=$SEQ_LEN_GGML overlap=$CHUNK_OVERLAP_GGML") - - val prefill = ArrayList(prefix.size + ids.size + suffix.size) - for (e in prefix) prefill.add(e) - for (id in ids) prefill.add(sumEmb(textEmbFromFull(id), codecPadEmb)) - for (e in suffix) prefill.add(e) - - val expectedSteps = ids.size * 4 - val cpuCap = 220 - val maxGen = minOf(expectedSteps * 2 + 20, cpuCap) - val eosBoostMinStep = (expectedSteps * 2) / 3 - - val tStart = System.currentTimeMillis() - var firstAudioLogged = false - val bvChan = kotlinx.coroutines.channels.Channel(capacity = 4) - val cfader = StreamingCrossfader({ pcm -> - if (!firstAudioLogged) { - nlog("ggml-stream seg $segIdx first audio at ${System.currentTimeMillis() - tStart}ms (${pcm.size} samples)") - firstAudioLogged = true - } - onAudio(pcm) - }, overlapTokens = CHUNK_OVERLAP_GGML) - - val consumerJob = kotlinx.coroutines.CoroutineScope(kotlinx.coroutines.Dispatchers.IO).launch { - try { - for (msg in bvChan) { - val tDec = System.currentTimeMillis() - // Le décodeur ggml accepte T variable. Codebooks chunk = [16][realTokens]. - val audio = ggml.decode(msg.codebooks, msg.realTokens) ?: ShortArray(0) - val dt = System.currentTimeMillis() - tDec - nlog("ggml-stream chunk T=${msg.realTokens} → ${audio.size} samples in ${dt}ms") - cfader.feedChunk(audio, msg.realTokens) - } - cfader.flushFinal() - } catch (e: Exception) { - nlog("ggml-stream seg $segIdx consumer error: ${e.message}") - } - } - - val collected = mutableListOf() - var nextChunkStart = 0 - - // Build chunk codebooks SANS padding : taille = realTokens (le décodeur - // ggml accepte T variable contrairement au V2 ONNX qui exige SEQ_LEN=60). - fun buildChunkCb(start: Int, real: Int): Array = Array(NUM_CODEBOOKS) { cb -> - IntArray(real) { t -> - val v = collected[start + t][cb] - if (v in 0 until CODEBOOK_SIZE) v else 0 - } - } - - try { - val onStep: (Int, IntArray) -> Unit = { _, codes -> - collected.add(codes) - while (collected.size >= nextChunkStart + SEQ_LEN_GGML) { - val cb = buildChunkCb(nextChunkStart, SEQ_LEN_GGML) - kotlinx.coroutines.runBlocking { bvChan.send(ChunkMsg(cb, SEQ_LEN_GGML)) } - nextChunkStart += EFFECTIVE_CHUNK_GGML - } - } - runInterleavedCpuFromEmbeds(prefill, emptyList(), maxGen, onStep, eosBoostMinStep = eosBoostMinStep) - } catch (e: Exception) { - nlog("ggml-stream seg $segIdx producer error: ${e.message}") - } - - // Trailing : tokens restants après le dernier chunk plein - val total = collected.size - if (total > nextChunkStart) { - val trailing = total - nextChunkStart - val cb = buildChunkCb(nextChunkStart, trailing) - kotlinx.coroutines.runBlocking { bvChan.send(ChunkMsg(cb, trailing)) } - } - bvChan.close() - consumerJob.join() - } - - /** - * Streaming variant of generateSegmentAudioVC. As the talker/CP loop - * produces codes step by step, BigVGAN chunks are dispatched on a - * background coroutine the moment SEQ_LEN codes are accumulated. For a - * 75-token segment this overlaps the last BigVGAN pass with the final - * ~20 talker/CP steps, cutting first-audio latency by ~4 s vs the - * sequential `generateSegmentAudioVC` path. - * - * Short segments ( Unit - ) { - if (bpeTokenizer == null || textEmbedsFullBuf == null || damienVoicePrefix == null || damienVoiceSuffix == null) { - nlog("generateSegmentAudioVCStreaming: Stage 2 assets missing"); return - } - if (!useCpuPath && (talkerPteModule == null || cpPteModule == null)) { - nlog("generateSegmentAudioVCStreaming: no talker backend (PTE & ggml-cpu both off)"); return - } - val prefix = damienVoicePrefix!! - val suffix = damienVoiceSuffix!! - val codecPadEmb = codecEmb(CODEC_PAD) - val ids = bpeTokenizer!!.encode(segText) - nlog("session seg $segIdx (stream) '${segText.take(60)}' → ${ids.size} tokens") - - val prefill = ArrayList(prefix.size + ids.size + suffix.size) - for (e in prefix) prefill.add(e) - for (id in ids) prefill.add(sumEmb(textEmbFromFull(id), codecPadEmb)) - for (e in suffix) prefill.add(e) - - val expectedSteps = (ids.size * 24) / 10 - val maxGen = minOf(expectedSteps * 3 / 2 + 10, MAX_CONTEXT - 15) - - val tStart = System.currentTimeMillis() - var firstAudioLogged = false - val bvChan = kotlinx.coroutines.channels.Channel(capacity = 4) - val cfader = StreamingCrossfader(onAudio = { pcm -> - if (!firstAudioLogged) { - nlog("streaming seg $segIdx first audio at ${System.currentTimeMillis() - tStart}ms (${pcm.size} samples)") - firstAudioLogged = true - } - onAudio(pcm) - }) - val consumerJob = kotlinx.coroutines.CoroutineScope(kotlinx.coroutines.Dispatchers.IO).launch { - try { - for (msg in bvChan) { - val quant = vqDecode(msg.codebooks) - val audio = runSpeechDecoderV2(quant) - cfader.feedChunk(audio, msg.realTokens) - } - cfader.flushFinal() - } catch (e: Exception) { - nlog("streaming seg $segIdx consumer error: ${e.message}") - } - } - - // Producer: run the interleaved talker/CP loop and dispatch each - // SEQ_LEN-aligned window of codes immediately. The consumer's - // crossfader holds back the last `overlapSamples` of audio per - // chunk, so the in-flight chunk's tail can still be mutated by the - // next chunk before being emitted; flushFinal() at end emits the - // last tail with fadeOut. End-of-stream is signalled by closing - // bvChan after the trailing partial chunk is sent. - val collected = mutableListOf() - var nextChunkStart = 0 - - fun buildChunkCb(start: Int, real: Int): Array = Array(NUM_CODEBOOKS) { cb -> - IntArray(SEQ_LEN) { t -> - val src = start + t - if (src < start + real) { - val v = collected[src][cb] - if (v in 0 until CODEBOOK_SIZE) v else 0 - } else 0 - } - } - - try { - val onStep: (Int, IntArray) -> Unit = { _, codes -> - collected.add(codes) - while (collected.size >= nextChunkStart + SEQ_LEN) { - val cb = buildChunkCb(nextChunkStart, SEQ_LEN) - kotlinx.coroutines.runBlocking { bvChan.send(ChunkMsg(cb, SEQ_LEN)) } - nextChunkStart += EFFECTIVE_CHUNK - } - } - if (useCpuPath) { - runInterleavedCpuFromEmbeds(prefill, emptyList(), maxGen, onStep) - } else { - runInterleavedPteFromEmbeds(prefill, emptyList(), maxGen, onStep) - } - } catch (e: Exception) { - nlog("streaming seg $segIdx producer error: ${e.message}") - } - - // Trailing chunk: any remaining tokens after the last full window - // (covers both the medium-segment partial-tail case and the - // short-segment nextChunkStart) { - val trailing = total - nextChunkStart - val cb = buildChunkCb(nextChunkStart, trailing) - kotlinx.coroutines.runBlocking { bvChan.send(ChunkMsg(cb, trailing)) } - } - bvChan.close() - consumerJob.join() - } - - /** - * Run the Hexagon talker + CP generation loop with a fully pre-built - * prefill (voice prefix + all text tokens). Same decode recipe as - * runInterleavedHexagon's inner loop: at each step the next talker - * input is codecSum (codec embedding of previous codes) + tts_pad - * (since all text has already been consumed in the prefill). On EOS, - * terminate early. Returns the generated [step, codebook] codes. - */ - private fun runHexGenWithPrefill( - prefill: List, - maxGen: Int, - // Floor on how many steps to generate before the dynamic boost is - // allowed to fire. Prevents short-text segments from terminating on - // a stray "EOS-leaning" hidden state during the first few frames. - // Callers pass ~50 % of expected speech length as a safety floor. - eosBoostMinStep: Int = -1, - // Once EOS rank falls below this threshold (model itself is - // "thinking about stopping"), start adding eosBoostScale per step - // until argmax flips to EOS. Empirically EOS rank plateaus ~150-700 - // mid-speech and dips to ~50-60 right at the natural end, so this - // catches the model's intent without a fixed length budget. - eosRankTrigger: Int = 60, - eosBoostScale: Float = 4.0f - ): Array { - val padE = ttsPadEmbed ?: return emptyArray() - val eosE = ttsEosEmbed ?: return emptyArray() - val allCodes = mutableListOf() - val generatedCb0 = mutableListOf() - var totalTalkerMs = 0L; var totalCpMs = 0L - - val tPrefill = System.currentTimeMillis() - val prefillResults = hexForward(prefill) - nlog("VC prefill (Hex): ${System.currentTimeMillis() - tPrefill}ms, ${prefillResults.size} steps") - if (prefillResults.isEmpty()) return emptyArray() - - var pastHidden = prefillResults.last().first - val prefillLogits = prefillResults.last().second - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) prefillLogits[j] = Float.NEGATIVE_INFINITY } - var currentCb0 = sampleTopK(prefillLogits, 0.9f, 50) - nlog("VC prefill done: first cb0=$currentCb0") - - // After the text has been fully consumed in prefill, Python's voice- - // clone loop feeds tts_eos once, then tts_pad for every subsequent - // decode step. We follow the same schedule so the model's attention - // sees the same "text exhausted" signal it was trained with. - var eosFedOnce = false - // Counter for the dynamic EOS boost — once armed, increments every - // step monotonically. Arming requires 3 CONSECUTIVE steps below - // the rank trigger (transient dips during normal speech aren't - // enough); this keeps short sentences from terminating mid-word - // on a fluke low rank. - var boostStepsActive = 0 - var consecLowRank = 0 - for (genStep in 0 until maxGen) { - val codes = IntArray(NUM_CODEBOOKS); codes[0] = currentCb0 - val tCp = System.currentTimeMillis() - val cpCodes = runCodePredictorInterleaved(pastHidden, currentCb0) - for (cb in 1 until NUM_CODEBOOKS) codes[cb] = cpCodes[cb - 1] - allCodes.add(codes); generatedCb0.add(currentCb0) - totalCpMs += System.currentTimeMillis() - tCp - - val codecSum = FloatArray(TALKER_DIM) - addEmb(codecSum, codecEmb(codes[0])) - for (cb in 1 until NUM_CODEBOOKS) addEmb(codecSum, cpEmb(cb - 1, codes[cb])) - val nextEmbed = if (!eosFedOnce) { eosFedOnce = true; sumEmb(codecSum, eosE) } else sumEmb(codecSum, padE) - - val tT = System.currentTimeMillis() - val results = hexForward(listOf(nextEmbed)) - totalTalkerMs += System.currentTimeMillis() - tT - if (results.isEmpty()) { nlog("VC: hex empty at step ${genStep+1}"); break } - pastHidden = results[0].first - val logits = results[0].second - for (j in CODEBOOK_SIZE until TALKER_VOCAB) { if (j != CODEC_EOS) logits[j] = Float.NEGATIVE_INFINITY } - val seen = HashSet(); for (prev in generatedCb0) seen.add(prev) - for (tok in seen) { logits[tok] = if (logits[tok] > 0) logits[tok] / 1.05f else logits[tok] * 1.05f } - - // Dynamic EOS detection: track current EOS rank, require 3 - // consecutive low-rank steps before arming. Single-step rank - // dips happen mid-utterance during low-energy phonemes and - // would otherwise trigger premature termination on short - // sentences. Once armed, the boost accumulates monotonically. - val eosLogit0 = logits[CODEC_EOS] - var eosRank = 0 - for (j in logits.indices) if (logits[j] > eosLogit0) eosRank++ - if (boostStepsActive == 0 && genStep >= eosBoostMinStep) { - if (eosRank < eosRankTrigger) { - consecLowRank++ - if (consecLowRank >= 3) { - nlog("VC boost armed at step ${genStep+1} (EOS rank $eosRank, 3 consecutive low)") - boostStepsActive = 1 - } - } else { - consecLowRank = 0 - } - } else if (boostStepsActive > 0) { - boostStepsActive++ - } - if (boostStepsActive > 0) { - logits[CODEC_EOS] += boostStepsActive * eosBoostScale - } - - var argmax = 0; var argmaxV = logits[0] - for (j in 1 until logits.size) if (logits[j] > argmaxV) { argmaxV = logits[j]; argmax = j } - if (argmax == CODEC_EOS) { nlog("VC EOS (boosted argmax) at step ${genStep+1}"); break } - - currentCb0 = sampleTopK(logits, 0.9f, 200) - if (currentCb0 == CODEC_EOS) { nlog("VC EOS (sampled) at step ${genStep+1}"); break } - - // Degeneracy guard #1: 9 consecutive identical cb0 → stop. - // Catches the simplest stuck loop. - val nHist = generatedCb0.size - if (nHist >= 9) { - val last = generatedCb0[nHist - 1] - var allSame = true - for (i in nHist - 9 until nHist) if (generatedCb0[i] != last) { allSame = false; break } - if (allSame && currentCb0 == last) { - nlog("VC degen: cb0=$last repeated ≥9× at step ${genStep+1}, stopping") - break - } - } - - // Degeneracy guard #2: low diversity in the recent window. - // The "page beg beg" filler doesn't repeat a single token — it - // cycles through 2-3 tokens. If the last 12 cb0 contain fewer - // than 4 unique values, the talker is in the cycle and we stop - // before the audio degrades further. - if (nHist >= 12) { - val recent = HashSet() - for (i in nHist - 12 until nHist) recent.add(generatedCb0[i]) - if (recent.size < 4) { - nlog("VC degen: only ${recent.size} unique cb0 in last 12 at step ${genStep+1}, stopping") - break - } - } - } - nlog("VC gen: ${allCodes.size} tokens | Talker(HEX): ${totalTalkerMs}ms | CP: ${totalCpMs}ms") - // Diagnostic: log full cb0 sequence so we can correlate against the - // page-beg region in the produced audio. - val cb0Str = generatedCb0.joinToString(",") - nlog("VC cb0 sequence: $cb0Str") - return allCodes.toTypedArray() - } - - /** - * Split text into short segments for the streaming pipeline. Reproduces - * the behaviour of scripts/prepare_tts_segments.py but on-device so the - * tablet doesn't depend on a PC-side preprocessor. The 120-character - * target matches the length where the talker still terminates reliably - * on EOS; longer segments risk the auto-repressor's repetition penalty - * cutting decode short of the full phrase. - */ - private fun splitSentences(text: String, maxChars: Int = 120): List { - val first = text.trim().split(Regex("(?<=[.!?;:])\\s+")) - val out = mutableListOf() - for (part in first) { - if (part.length <= maxChars) { - if (part.isNotBlank()) out.add(part.trim()) - continue - } - // Break overlong sentences at commas, greedily packing sub-parts - // back together up to maxChars so we don't over-split. - val subs = part.split(Regex("(?<=,)\\s+")) - var current = "" - for (s in subs) { - if (current.isNotEmpty() && current.length + s.length > maxChars) { - out.add(current.trim()); current = s - } else { - current = if (current.isEmpty()) s else "$current $s" - } - } - if (current.isNotBlank()) out.add(current.trim()) - } - return if (out.isEmpty()) listOf(text) else out - } - - /** - * Tokenize `text` on-device and stream the synthesized audio segment by - * segment via `onSegmentReady`. The PC-side prep script becomes optional — - * this is the first path in the app where TTS runs fully offline on - * arbitrary LLM output. - * - * For each segment: - * 1. BPE tokenize the segment text (same algorithm as Python's - * Qwen2Tokenizer). - * 2. Look up each token ID in the fp16 full-vocab table, converted to - * fp32 on the fly. One embedding per token. - * 3. Call the existing runInterleavedHexagon loop — which already - * synthesizes its own decode inputs via codec_sum + trailing text — - * so we reuse the same prefill construction and generation path that - * runs today for the pre-computed-embeds test harness. - * 4. Decode codes → audio via decodeChunked, emit the audio through - * the callback immediately, save WAV per segment plus the concat. - * - * Emits at most one callback per segment. First-audio latency ≈ prefill + - * one segment's decode (typically ~17-22s for a 5-6s-duration segment on - * Snapdragon 8 Elite). The ordering and gaps between segments are the - * same as generateFromEmbedsHexagonStreaming. - */ - fun synthesizeTextStreaming( - text: String, - onSegmentReady: ((segIdx: Int, audio: ShortArray) -> Unit)? = null - ): ShortArray { - if (!loaded || !useHexagonTalker) { - nlog("synthesizeTextStreaming: Hexagon talker not ready"); return ShortArray(0) - } - if (bpeTokenizer == null || textEmbedsFullBuf == null) { - nlog("synthesizeTextStreaming: Stage 2 assets missing"); return ShortArray(0) - } - val segments = splitSentences(text) - nlog("synthesizeTextStreaming: ${segments.size} segment(s) for ${text.length} chars") - - hexReset() - val segmentAudios = mutableListOf() - // Wider gap (250 ms) between segments — gives the listener a small - // breath/pause that mirrors how real speakers separate sentences, - // and masks the EOS-boost cut by surrounding it with silence rather - // than another sentence's onset. - val gapSamples = SR * 250 / 1000 - val gap = ShortArray(gapSamples) - val t0 = System.currentTimeMillis() - - val prefix = damienVoicePrefix!! - val suffix = damienVoiceSuffix!! - // CODEC_PAD embedding is the per-token companion that Python voice- - // cloning sums into every text-encoded prefill position. Computed - // once here so the per-token loop stays a simple vector add. - val codecPadEmb = codecEmb(CODEC_PAD) - - for ((segIdx, segText) in segments.withIndex()) { - if (segIdx > 0) hexReset() - val tSeg = System.currentTimeMillis() - val ids = bpeTokenizer!!.encode(segText) - nlog("Seg ${segIdx+1}/${segments.size}: '${segText.take(60)}' → ${ids.size} tokens: ${ids.toList()}") - - // Voice-cloning prefill, fully reconstructed on-device — exact - // structure Python emits via generate_voice_clone (verified - // bit-for-bit by comparing captured Baer segments): - // [0..8] damienVoicePrefix (9 fixed positions, xvector@7) - // [9..N-3] text_projection(BPE_id) + codec_embedding(CODEC_PAD) - // [N-2, N-1] damienVoiceSuffix (2 fixed positions, end-of-text marker) - // The earlier attempt skipped the suffix and used raw text - // projections, which produced garbled audio — the talker needs - // BOTH the per-token codec_pad sum AND the closure markers to - // know that text input has ended and decoding can begin. - val prefill = ArrayList(prefix.size + ids.size + suffix.size) - for (e in prefix) prefill.add(e) - for (id in ids) prefill.add(sumEmb(textEmbFromFull(id), codecPadEmb)) - for (e in suffix) prefill.add(e) - - // Generous absolute cap; the dynamic EOS boost (triggered when - // the model's own EOS rank dips below threshold) is what - // actually terminates generation. The minStep floor protects - // against early-termination spikes for short sentences. - val expectedSteps = (ids.size * 24) / 10 // ids * 2.4 (int math) - val maxGen = minOf(expectedSteps * 3 / 2 + 10, MAX_CONTEXT - 15) - val eosBoostMinStep = expectedSteps / 2 - val codes = runHexGenWithPrefill(prefill, maxGen, eosBoostMinStep) - if (codes.isEmpty()) { nlog("Seg ${segIdx+1}: empty codes"); continue } - - val n = codes.size - val padLen = maxOf(n, SEQ_LEN) - val codebooks = Array(NUM_CODEBOOKS) { cb -> - IntArray(padLen) { t -> - if (t < n) { val v = codes[t][cb]; if (v in 0 until CODEBOOK_SIZE) v else 0 } else 0 - } - } - val rawAudio = decodeChunked(codebooks, n) - // Cosine fade-out on the last 40 ms — softens the cut imposed by - // the EOS boost so the segment ends on a recognisable phoneme - // tail instead of an abrupt sample-clip. - val audio = fadeOut(rawAudio, 40) - val segMs = System.currentTimeMillis() - tSeg - val budgetHit = if (n >= maxGen) " [maxGen cap]" else "" - nlog("Seg ${segIdx+1}/${segments.size}: $n tokens, ${audio.size/SR.toFloat()}s audio in ${segMs}ms$budgetHit") - - segmentAudios.add(audio) - saveWav("/data/local/tmp/kazeia/kazeia_stream_seg${segIdx+1}.wav", audio) - onSegmentReady?.invoke(segIdx, audio) - } - - if (segmentAudios.isEmpty()) return ShortArray(0) - val total = segmentAudios.sumOf { it.size } + maxOf(0, segmentAudios.size - 1) * gapSamples - val concat = ShortArray(total) - var off = 0 - for ((i, s) in segmentAudios.withIndex()) { - System.arraycopy(s, 0, concat, off, s.size); off += s.size - if (i < segmentAudios.size - 1) { System.arraycopy(gap, 0, concat, off, gapSamples); off += gapSamples } - } - saveWav("/data/local/tmp/kazeia/kazeia_stream_full.wav", concat) - nlog("synthesizeTextStreaming total: ${System.currentTimeMillis() - t0}ms for ${concat.size/SR.toFloat()}s") - return concat - } - - /** - * Trim low-energy tail from a voice-cloned segment. When the talker - * fails to emit EOS within maxGen, the remaining decoded codes tend - * to be "page beg beg" fillers — audible as a mumbled tail. This - * RMS-based trim finds the last sustained high-energy window and cuts - * after it, with a small fade-out so the last real syllable keeps its - * natural decay. Extracted from generateMultiSegment so the - * synthesizeTextStreaming path can reuse it verbatim. - */ - private fun trimTailLowEnergy(audio: ShortArray): ShortArray { - if (audio.size < SR / 2) return audio - val winSamples = SR * 40 / 1000 // 40 ms windows = 960 samples - val nWin = audio.size / winSamples - if (nWin < 10) return audio - val rms = FloatArray(nWin) - for (w in 0 until nWin) { - var s = 0.0 - val o = w * winSamples - for (i in 0 until winSamples) { val x = audio[o + i].toFloat() / 32768f; s += x * x } - rms[w] = kotlin.math.sqrt(s / winSamples).toFloat() - } - var peak = 0f - for (w in 0 until nWin) if (rms[w] > peak) peak = rms[w] - - // Heuristic 1: classic 35 % sustained threshold from the back. - val thrSust = peak * 0.35f - var lastSpeech = nWin - 1 - for (w in nWin - 1 downTo 2) { - if (rms[w] >= thrSust && rms[w-1] >= thrSust && rms[w-2] >= thrSust) { - lastSpeech = w; break - } - } - - // Heuristic 2: "low-energy tail" — the "page beg beg" filler emits - // audio that's louder than silence but quieter than real speech. - // If the last 30 % of the segment has a max RMS under 40 % of the - // global peak, the tail is degenerate; cut at the last sustained- - // speech window before the tail starts. - val tailStart = (nWin * 7) / 10 - var tailMax = 0f - for (w in tailStart until nWin) if (rms[w] > tailMax) tailMax = rms[w] - if (tailMax < peak * 0.40f) { - for (w in tailStart - 1 downTo 2) { - if (rms[w] >= thrSust && rms[w-1] >= thrSust && rms[w-2] >= thrSust) { - if (w < lastSpeech) lastSpeech = w - break - } - } - } - - val keepWin = (lastSpeech + 2).coerceAtMost(nWin - 1) - var keepSamples = (keepWin + 1) * winSamples - // Over-trim guard: never cut below 40 % of the raw length. - val minKeep = (audio.size * 4) / 10 - if (keepSamples < minKeep) keepSamples = minKeep - return audio.copyOf(keepSamples) - } - - /** - * Apply a short cosine fade-out to the tail of an audio buffer. The - * EOS boost ends generation right at the model's chosen stop point, - * which usually clips the natural decay of the last syllable. A - * 40-ms fade smooths that into a recognisable phoneme tail without - * shortening the perceived word. - */ - private fun fadeOut(audio: ShortArray, fadeMs: Int = 40): ShortArray { - val fadeSamples = SR * fadeMs / 1000 - if (audio.size <= fadeSamples) return audio - val out = audio.copyOf() - val start = out.size - fadeSamples - for (i in 0 until fadeSamples) { - // Cosine roll-off: 1 → 0 over fadeSamples - val t = i.toFloat() / fadeSamples - val gain = 0.5f * (1f + kotlin.math.cos(Math.PI.toFloat() * t)) - out[start + i] = (out[start + i] * gain).toInt().toShort() - } - return out - } - - /** Write PCM16 mono audio to a WAV file. Used by the streaming pipeline to - * save one file per segment plus the concatenated result for inspection. */ - private fun saveWav(path: String, audio: ShortArray) { - try { - val fos = java.io.FileOutputStream(path) - val dataLen = audio.size * 2 - val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN) - header.put("RIFF".toByteArray()); header.putInt(36 + dataLen) - header.put("WAVE".toByteArray()); header.put("fmt ".toByteArray()) - header.putInt(16); header.putShort(1); header.putShort(1) - header.putInt(SR); header.putInt(SR * 2); header.putShort(2); header.putShort(16) - header.put("data".toByteArray()); header.putInt(dataLen) - fos.write(header.array()) - val buf = ByteBuffer.allocate(dataLen).order(ByteOrder.LITTLE_ENDIAN) - for (s in audio) buf.putShort(s) - fos.write(buf.array()); fos.close() - } catch (e: Exception) { nlog("WAV save failed ($path): ${e.message}") } - } - - /** Test with pre-computed codec tokens from PC (for validation) */ - fun testWithPrecomputedCodes(codesPath: String, realTokens: Int = 16): ShortArray { - if (!loaded) return ShortArray(0) - nlog("Testing with pre-computed codes: $codesPath (realTokens=$realTokens)") - try { - val bytes = File(codesPath).readBytes() - val bb = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) - val allCodes = Array(NUM_CODEBOOKS) { IntArray(SEQ_LEN) } - for (cb in 0 until NUM_CODEBOOKS) { - for (t in 0 until SEQ_LEN) { - allCodes[cb][t] = bb.int.coerceIn(0, CODEBOOK_SIZE - 1) - } - } - nlog("Loaded ${NUM_CODEBOOKS} × ${SEQ_LEN} codes") - nlog("codes[0][:5] = ${allCodes[0].take(5).toList()}") - nlog("codes[1][:5] = ${allCodes[1].take(5).toList()}") - val t0 = System.currentTimeMillis() - val quantized = vqDecode(allCodes) - // Dump quantized stats for debug - var qMin = Float.MAX_VALUE; var qMax = Float.MIN_VALUE; var qSum = 0.0 - for (v in quantized) { qMin = minOf(qMin, v); qMax = maxOf(qMax, v); qSum += v * v } - nlog("quantized: size=${quantized.size}, range=[${qMin}, ${qMax}], rms=${Math.sqrt(qSum / quantized.size)}") - // Save quantized to file for comparison - try { - val qf = java.io.File("/data/local/tmp/kazeia/quantized_dump.bin") - qf.writeBytes(java.nio.ByteBuffer.allocate(quantized.size * 4).order(java.nio.ByteOrder.LITTLE_ENDIAN).also { - for (v in quantized) it.putFloat(v) - }.array()) - nlog("Saved quantized to ${qf.absolutePath}") - } catch (_: Exception) {} - val fullAudio = runSpeechDecoder(quantized) - val trimSamples = minOf(realTokens * SAMPLES_PER_TOKEN, fullAudio.size) - val audio = fullAudio.copyOf(trimSamples) - nlog("Decode: ${System.currentTimeMillis() - t0}ms, audio=${audio.size.toFloat()/SR}s (trimmed from ${fullAudio.size.toFloat()/SR}s)") - return audio - } catch (e: Exception) { - nlog("Test error: ${e.message}") - return ShortArray(0) - } - } - - override fun release() { - stop() - // Free ggml-cpu Talker+CP handles - if (talkerCpuHandle != 0L) { try { TtsTalkerCpuJni.nativeFree(talkerCpuHandle) } catch (_: Exception) {}; talkerCpuHandle = 0L } - if (cpCpuHandle != 0L) { try { TtsCpCpuJni.nativeFree(cpCpuHandle) } catch (_: Exception) {}; cpCpuHandle = 0L } - useCpuPath = false - // Stop hexagon runners cleanly - if (useHexagonTalker || useHexagonCp) { - hexStopRunner() - } - talkerKv?.close() - cpKv?.close() - preConv?.close(); preprocessor?.close(); convDecoder?.close() - ortEnv?.close() - loaded = false - } -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsGgmlDecoder.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsGgmlDecoder.kt deleted file mode 100644 index 432bb8a..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/Qwen3TtsGgmlDecoder.kt +++ /dev/null @@ -1,91 +0,0 @@ -package com.kazeia.tts - -import android.util.Log - -/** - * Décodeur Qwen3-TTS pur ggml C++. Remplace la voie .pte ExecuTorch. - * - * Pipeline complet codes [16, T] long → wav [T*1920] PCM16 fait en C++ : - * Quantizer + pre_conv + pre_transformer + 2× upsample + BigVGAN. - * - * Validé bit-close vs PyTorch reference (max_abs_diff 2.1e-3, RMSE 1.0e-4) - * sur Damien FR. Compute ~4.85s pour 4.64s audio sur OnePlus Pad 3 8 threads - * (RTF 1.05) — équivalent à ExecuTorch CPU XNNPACK / 2 grâce aux optims - * ARM features (i8mm, dotprod, fp16) + F16 conv weights + layout uniforme. - */ -class Qwen3TtsGgmlDecoder(private val ggufPath: String) { - - companion object { - private const val TAG = "TtsDecoderGgml" - init { - try { - System.loadLibrary("tts_decoder_ggml") - Log.i(TAG, "loaded libtts_decoder_ggml.so") - } catch (e: UnsatisfiedLinkError) { - Log.e(TAG, "loadLibrary failed: ${e.message}") - } - } - } - - private var handle: Long = 0L - - fun load(): Boolean { - if (handle != 0L) return true - // Test 2026-05-03 : GGML_NUM_THREADS=4 ne réduisait PAS le pic - // (ggml fait du work-sharing sur tensors uniques, pas de réplication - // par thread). Reverté → laisse default 8 (plus rapide). - handle = nativeCreate(ggufPath) - if (handle == 0L) { - Log.e(TAG, "nativeCreate returned 0 (load failed)") - return false - } - Log.i(TAG, "decoder loaded from $ggufPath, handle=0x${java.lang.Long.toHexString(handle)}") - return true - } - - /** - * Décode codes [n_q=16][T] (codebooks par layer × timesteps) → PCM16 [T*1920]. - * - * @param codebooks Tableau IntArray par codebook, chaque codebook contient T indices [0, 2047]. - * @param numTokens Nombre de timesteps réels T (= taille de chaque codebook). - * @return ShortArray de T*1920 samples PCM16 à 24 kHz, ou null si échec. - */ - fun decode(codebooks: Array, numTokens: Int): ShortArray? { - if (handle == 0L) { - Log.e(TAG, "decode called before load()") - return null - } - val nQ = codebooks.size - if (nQ != 16) { - Log.e(TAG, "Expected 16 codebooks, got $nQ") - return null - } - // Aplatir en codes_flat[n_q * T] avec layout codes_flat[layer*T + t] - val flat = IntArray(nQ * numTokens) - for (layer in 0 until nQ) { - for (t in 0 until numTokens) { - flat[layer * numTokens + t] = codebooks[layer][t] - } - } - val t0 = System.currentTimeMillis() - val pcm = nativeDecode(handle, flat, numTokens) - if (pcm == null) { - Log.e(TAG, "nativeDecode returned null") - return null - } - val dt = System.currentTimeMillis() - t0 - Log.i(TAG, "decode T=$numTokens → ${pcm.size} samples in ${dt}ms (RTF ${"%.2f".format(dt / (pcm.size / 24.0))})") - return pcm - } - - fun close() { - if (handle != 0L) { - nativeDestroy(handle) - handle = 0L - } - } - - private external fun nativeCreate(ggufPath: String): Long - private external fun nativeDecode(handle: Long, codes: IntArray, T: Int): ShortArray? - private external fun nativeDestroy(handle: Long) -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsCpCpuJni.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/TtsCpCpuJni.kt deleted file mode 100644 index 5643c9c..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsCpCpuJni.kt +++ /dev/null @@ -1,51 +0,0 @@ -package com.kazeia.tts - -/** - * JNI bridge to the in-process CP CPU runner (libtts_cp_cpu.so). - * - * Replaces the .pte ExecuTorch CP path with a llama.cpp-based forward that - * runs cp_f16.gguf in CPU NEON mode (n_gpu_layers=0). - * - * Heads + codec embeddings (15 × 2048 × 1024 fp32, ~125 MB each) loaded - * separately via [nativeInit]. They live in /data/local/tmp/kazeia/models/ - * (cp_heads.bin, cp_codec_embs.bin) and are mmap-friendly fp32 row-major. - * - * Per Talker step : [nativeForward] runs 17 forward decodes + 15 NEON - * dot_argmax classifications → returns 15 codebook codes (cb1..cb15). - * - * KV is RECREATED per call (matches tts-cp-runner.cpp ref to avoid Hexagon - * NaN — TODO optim Phase 4 : try memory_clear instead since CPU doesn't - * have the NaN issue, save the recreate cost). - */ -object TtsCpCpuJni { - - init { - // Order matters : ggml deps before llama. Talker JNI loads them - // first, but if CP loads alone we ensure them too. - System.loadLibrary("ggml-base") - System.loadLibrary("ggml-cpu") - System.loadLibrary("ggml") - System.loadLibrary("llama") - System.loadLibrary("tts_cp_cpu") - } - - /** Load cp_f16.gguf + heads + codec embeddings. - * - modelPath : path to cp_f16.gguf (~166 MB) - * - headsPath : path to cp_heads.bin (15 × 2048 × 1024 fp32 = 125 MB) - * - embsPath : path to cp_codec_embs.bin (125 MB) - * Returns native handle, 0 on failure. */ - @JvmStatic external fun nativeInit(modelPath: String, headsPath: String, embsPath: String): Long - - /** Run one CP step. - * - hidden : Talker hidden state output [1024] fp32 - * - cb0Emb : codec embedding for predicted cb0 [1024] fp32 - * Returns 15 int32 codes (cb1..cb15), null on failure. */ - @JvmStatic external fun nativeForward(handle: Long, hidden: FloatArray, cb0Emb: FloatArray): IntArray? - - /** Reset KV cache. Each forward recreates ctx anyway, but explicit - * reset between sentences is supported. */ - @JvmStatic external fun nativeReset(handle: Long) - - /** Free model + ctx + heads + embs buffers. */ - @JvmStatic external fun nativeFree(handle: Long) -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/TtsEngine.kt deleted file mode 100644 index f803a66..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsEngine.kt +++ /dev/null @@ -1,94 +0,0 @@ -package com.kazeia.tts -// Engine TTS Qwen3 in-process (libkazeia_tts.so). Load une fois, synthesize N fois. -// Le bridge natif est dans dist/jni/kazeia_tts_jni.cpp ; build via dist/build_kazeia_tts.sh -// ou dist/CMakeLists.txt. -// -// Empreinte RAM typique : ~3.5 GB (talker f32 ~1.6 GB + CP f16 mixte ~250 MB + -// decoder ~325 MB + vocab Qwen3 ~50 MB + fixtures text_embed/tp_* ~1.2 GB). - -class TtsJni { - external fun nativeLoad( - talkerGguf: String, - vocabGguf: String, - dumpDir: String, - useHtp: Boolean, - nThreads: Int, - cpUseCache: Boolean - ): Long - - external fun nativeSynthesize( - handle: Long, - text: String, - outWavPath: String, - maxSteps: Int, - seed: Int, - cpTemp: Float, cpTopK: Int, cpTopP: Float, cpRepPenalty: Float, - talkerTemp: Float, talkerTopK: Int, talkerTopP: Float, talkerRepPenalty: Float - ): IntArray // [err, frames, total_ms, prefill_ms, talker_ms, cp_ms, decoder_ms] - - // Dérive un x_vector depuis un WAV (downmix + resample à 24k côté natif) et - // le mémorise comme voix courante pour ce handle. Effet au prochain synth. - external fun nativeSetVoiceWav(handle: Long, wavPath: String): Boolean - - external fun nativeFree(handle: Long) - - companion object { init { System.loadLibrary("kazeia_tts") } } -} - -data class TtsResult( - val frames: Int, - val audioMs: Int, - val totalMs: Int, - val prefillMs: Int, - val talkerMs: Int, - val cpMs: Int, - val decoderMs: Int -) - -// Configuration sampling. Défauts validés sur fixture FR (cf project_tts_session_28may). -data class TtsSampling( - // Défauts validés à l'écoute (top_p=0.95 + rep=1.10 moins robotique vs 1.0/1.05). - val cpTemp: Float = 0.9f, val cpTopK: Int = 50, val cpTopP: Float = 0.95f, val cpRepPenalty: Float = 1.10f, - val talkerTemp: Float = 0.9f, val talkerTopK: Int = 50, val talkerTopP: Float = 0.95f, val talkerRepPenalty: Float = 1.10f -) - -class TtsEngine( - talkerGguf: String, - vocabGguf: String, - dumpDir: String, - useHtp: Boolean = false, - nThreads: Int = 6, - cpUseCache: Boolean = true -) { - private val jni = TtsJni() - private val h = jni.nativeLoad(talkerGguf, vocabGguf, dumpDir, useHtp, nThreads, cpUseCache) - init { require(h != 0L) { "TtsEngine: load FAILED (talker=$talkerGguf, vocab=$vocabGguf, dump=$dumpDir)" } } - - @Throws(IllegalStateException::class) - fun synthesize( - text: String, - outWavPath: String, - maxSteps: Int = 256, - seed: Int = 42, - sampling: TtsSampling = TtsSampling() - ): TtsResult { - val r = jni.nativeSynthesize( - h, text, outWavPath, maxSteps, seed, - sampling.cpTemp, sampling.cpTopK, sampling.cpTopP, sampling.cpRepPenalty, - sampling.talkerTemp, sampling.talkerTopK, sampling.talkerTopP, sampling.talkerRepPenalty - ) - if (r[0] != 0) error("TtsEngine.synthesize err=${r[0]} for text=\"$text\"") - val frames = r[1] - return TtsResult( - frames = frames, - audioMs = frames * 1000 / 12, // 12 frames/s = 80 ms par frame - totalMs = r[2], - prefillMs = r[3], - talkerMs = r[4], - cpMs = r[5], - decoderMs = r[6] - ) - } - - fun release() { jni.nativeFree(h) } -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsPipeline.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/TtsPipeline.kt deleted file mode 100644 index 2ba802f..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsPipeline.kt +++ /dev/null @@ -1,28 +0,0 @@ -package com.kazeia.tts - -/** Native C++ TTS pipeline: talker + CP loop with zero Java overhead. */ -object TtsPipeline { - init { System.loadLibrary("tts_pipeline") } - - /** Load and warmup both .pte models. Returns true on success. */ - external fun nativeInit(talkerPath: String, cpPath: String): Boolean - - /** Release native resources. */ - external fun nativeDestroy() - - /** - * Run full pipeline. Returns flat int array: [numTokens × 16] codebook codes. - * Each group of 16 ints = [CB0, CB1, ..., CB15] for one time step. - */ - external fun nativeRun( - prefillEmbeds: FloatArray, nPrefill: Int, - trailingEmbeds: FloatArray?, nTrailing: Int, - codecEmbedding: FloatArray, - cpEmbeddings: FloatArray, - cpHeads: FloatArray, - talkerCos: FloatArray, talkerSin: FloatArray, - cpCos: FloatArray, cpSin: FloatArray, - eosEmbed: FloatArray, padEmbed: FloatArray, - maxTokens: Int - ): IntArray? -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsTalkerCpuJni.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/TtsTalkerCpuJni.kt deleted file mode 100644 index 5faa607..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/tts/TtsTalkerCpuJni.kt +++ /dev/null @@ -1,40 +0,0 @@ -package com.kazeia.tts - -/** - * JNI bridge to the in-process Talker CPU runner (libtts_talker_cpu.so). - * - * Replaces the .pte ExecuTorch path with a llama.cpp-based forward that - * runs talker_f16.gguf in CPU NEON mode (n_gpu_layers=0). - * - * Lifecycle: [init] once at boot → [forward] per autoregressive step → - * [reset] between sentences → [free] at TTS engine release. - * - * Mesured POC (2026-04-29) : - * ms/step CPU NEON ≈ 17 (vs 43 .pte NPU) → ×2.5 plus rapide - * Cible RTF projeté < 1 → "beg beg beg" éliminé structurellement - */ -object TtsTalkerCpuJni { - - init { - // Order matters : ggml deps before llama - System.loadLibrary("ggml-base") - System.loadLibrary("ggml-cpu") - System.loadLibrary("ggml") - System.loadLibrary("llama") - System.loadLibrary("tts_talker_cpu") - } - - /** Load talker_f16.gguf in CPU mode. Returns native handle, 0 on failure. */ - @JvmStatic external fun nativeInit(modelPath: String): Long - - /** Run one forward step. embd is 1024 fp32. Returns 4096 fp32 : - * [0..1023] = hidden state, [1024..4095] = cb0 logits over codec vocab. - * null on failure. */ - @JvmStatic external fun nativeForward(handle: Long, embd: FloatArray): FloatArray? - - /** Reset KV cache for a new sentence. */ - @JvmStatic external fun nativeReset(handle: Long) - - /** Free model + ctx. */ - @JvmStatic external fun nativeFree(handle: Long) -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/v2/ChatActivityV2.kt b/kazeia-android/app/src/main/java/com/kazeia/v2/ChatActivityV2.kt deleted file mode 100644 index e1aeec0..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/v2/ChatActivityV2.kt +++ /dev/null @@ -1,331 +0,0 @@ -package com.kazeia.v2 - -import android.Manifest -import android.content.ComponentName -import android.content.Context -import android.content.Intent -import android.content.ServiceConnection -import android.content.pm.PackageManager -import android.graphics.Color -import android.os.Bundle -import android.os.IBinder -import android.view.Gravity -import android.view.ViewGroup.LayoutParams.MATCH_PARENT -import android.view.ViewGroup.LayoutParams.WRAP_CONTENT -import android.graphics.Typeface -import android.widget.Button -import android.widget.LinearLayout -import android.widget.ScrollView -import android.widget.TextView -import androidx.activity.result.contract.ActivityResultContracts -import androidx.appcompat.app.AppCompatActivity -import androidx.lifecycle.lifecycleScope -import kotlinx.coroutines.launch - -/** - * UI test page Phase 1 du rewrite Kazeia v2. - * - * - Bouton "Parler" déclenche la capture micro avec VAD Silero ; pas besoin - * d'arrêter manuellement, VAD termine le segment automatiquement. - * - Bulle USER (transcription STT en bleu, alignée droite) - * - Bulle KAZEIA (dots animés "..." ou texte final, alignée gauche) - * - Status bar bas affiche l'état du pipeline - * - * Phase 1 = STT (VAD + Whisper). Phase 2 ajoutera LLM cascade, Phase 3 TTS. - */ -class ChatActivityV2 : AppCompatActivity() { - - private var service: KazeiaServiceV2? = null - private var bound = false - - private lateinit var statusView: TextView - private lateinit var bubbleUser: TextView - private lateinit var bubbleKazeia: TextView - private lateinit var pttButton: Button - private lateinit var bubblesContainer: LinearLayout - private lateinit var logsView: TextView - private lateinit var logsScroll: ScrollView - private val logsBuffer = StringBuilder(8192) - private val logsMaxLines = 400 - - private lateinit var metricsView: MetricsGraphView - private val metricsSampler = MetricsSampler { cpu, ram, avail -> - if (::metricsView.isInitialized) metricsView.addSample(cpu, ram, avail) - } - - private val permLauncher = registerForActivityResult(ActivityResultContracts.RequestPermission()) { - // No-op, user just sees state if denied - } - - private val connection = object : ServiceConnection { - override fun onServiceConnected(name: ComponentName?, binder: IBinder?) { - service = (binder as KazeiaServiceV2.LocalBinder).service() - bound = true - KazeiaLogger.pipeline("ChatActivityV2 bound to KazeiaServiceV2") - observeService() - } - override fun onServiceDisconnected(name: ComponentName?) { - service = null - bound = false - } - } - - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - - // Permissions - if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { - permLauncher.launch(Manifest.permission.RECORD_AUDIO) - } - - // Layout split horizontal : - // gauche (60 %) : chat (titre + bulles + bouton + status) - // droite (40 %) : panneau logs en monospace, auto-scroll bas - val root = LinearLayout(this).apply { - orientation = LinearLayout.HORIZONTAL - setPadding(16, 16, 16, 16) - setBackgroundColor(Color.parseColor("#FAFAFA")) - } - - // ─── Colonne gauche : chat ─────────────────────────────────── - val leftCol = LinearLayout(this).apply { - orientation = LinearLayout.VERTICAL - setPadding(8, 8, 16, 8) - layoutParams = LinearLayout.LayoutParams(0, MATCH_PARENT, 0.6f) - } - - val title = TextView(this).apply { - text = "Kazeia v2" - textSize = 20f - setTextColor(Color.parseColor("#222222")) - setPadding(0, 0, 0, 16) - } - leftCol.addView(title) - - bubblesContainer = LinearLayout(this).apply { - orientation = LinearLayout.VERTICAL - } - bubbleUser = makeBubble(isUser = true).apply { text = "" } - bubbleKazeia = makeBubble(isUser = false).apply { text = "" } - bubblesContainer.addView(bubbleUser) - bubblesContainer.addView(bubbleKazeia) - - val chatScroll = ScrollView(this).apply { - addView(bubblesContainer) - layoutParams = LinearLayout.LayoutParams(MATCH_PARENT, 0, 1f) - } - leftCol.addView(chatScroll) - - pttButton = Button(this).apply { - text = "▶ Parler (VAD)" - textSize = 18f - setOnClickListener { onPttClicked() } - } - leftCol.addView(pttButton) - - statusView = TextView(this).apply { - text = "Status: Idle (service en chargement)" - textSize = 12f - setTextColor(Color.GRAY) - setPadding(0, 12, 0, 0) - } - leftCol.addView(statusView) - - root.addView(leftCol) - - // ─── Colonne droite : panneau logs ─────────────────────────── - val rightCol = LinearLayout(this).apply { - orientation = LinearLayout.VERTICAL - setPadding(16, 8, 8, 8) - setBackgroundColor(Color.parseColor("#0D1117")) - layoutParams = LinearLayout.LayoutParams(0, MATCH_PARENT, 0.4f) - } - - // Graph CPU/RAM en haut (hauteur fixe 200 dp) - metricsView = MetricsGraphView(this).apply { - layoutParams = LinearLayout.LayoutParams(MATCH_PARENT, (200 * resources.displayMetrics.density).toInt()) - } - rightCol.addView(metricsView) - - val logsTitle = TextView(this).apply { - text = "Logs pipeline (live)" - textSize = 14f - setTextColor(Color.parseColor("#8B949E")) - setTypeface(Typeface.MONOSPACE, Typeface.BOLD) - setPadding(8, 8, 8, 8) - } - rightCol.addView(logsTitle) - - logsView = TextView(this).apply { - text = "" - textSize = 10f - setTypeface(Typeface.MONOSPACE) - setTextColor(Color.parseColor("#C9D1D9")) - setBackgroundColor(Color.parseColor("#0D1117")) - setPadding(8, 8, 8, 8) - setHorizontallyScrolling(false) - // Wrap long lines - } - logsScroll = ScrollView(this).apply { - addView(logsView) - layoutParams = LinearLayout.LayoutParams(MATCH_PARENT, 0, 1f) - } - rightCol.addView(logsScroll) - - // Bouton clear logs - val clearBtn = Button(this).apply { - text = "Clear logs" - textSize = 12f - setOnClickListener { - logsBuffer.clear() - logsView.text = "" - } - } - rightCol.addView(clearBtn) - - root.addView(rightCol) - - setContentView(root) - - // Stream les logs depuis KazeiaLogger - observeLogs() - // Démarre l'échantillonnage CPU/RAM (500 ms) - metricsSampler.start() - - // Bind seul — BIND_AUTO_CREATE crée le service automatiquement. - // Pas besoin de startService (qui demanderait ForegroundService permission). - val intent = Intent(this, KazeiaServiceV2::class.java) - bindService(intent, connection, Context.BIND_AUTO_CREATE) - } - - private fun makeBubble(isUser: Boolean): TextView { - return TextView(this).apply { - textSize = 16f - setPadding(24, 16, 24, 16) - val lp = LinearLayout.LayoutParams(WRAP_CONTENT, WRAP_CONTENT) - lp.setMargins(if (isUser) 80 else 0, 8, if (isUser) 0 else 80, 8) - lp.gravity = if (isUser) Gravity.END else Gravity.START - layoutParams = lp - setBackgroundColor(if (isUser) Color.parseColor("#3B82F6") else Color.parseColor("#E5E7EB")) - setTextColor(if (isUser) Color.WHITE else Color.parseColor("#111111")) - } - } - - private fun observeService() { - val s = service ?: return - lifecycleScope.launch { - s.state.collect { st -> renderState(st) } - } - lifecycleScope.launch { - s.events.collect { ev -> handleEvent(ev) } - } - } - - private fun observeLogs() { - lifecycleScope.launch { - KazeiaLogger.logFlow.collect { entry -> - appendLog(entry) - } - } - } - - private fun appendLog(entry: KazeiaLogger.LogEntry) { - // Plain text + couleur seulement sur le tag via SpannableStringBuilder. - // Évite Html.fromHtml(fullBuffer) qui allouait des MB par log line. - val tagColor = when { - entry.level == KazeiaLogger.Level.E -> 0xFFFF7B72.toInt() // rouge - entry.tag == "PIPELINE" -> 0xFF7EE787.toInt() - entry.tag == "STT" -> 0xFF79C0FF.toInt() - entry.tag == "THINKER" || entry.tag == "SPEAKER" -> 0xFFFFA657.toInt() - entry.tag.startsWith("TTS-") -> 0xFFD2A8FF.toInt() - else -> 0xFFC9D1D9.toInt() - } - val plain = "${entry.ts} ${entry.tag} ${entry.msg}\n" - logsBuffer.append(plain) - - // Trim si > 16 KB (garde les ~250 dernières lignes) - if (logsBuffer.length > 16_384) { - val cutPoint = logsBuffer.indexOf("\n", logsBuffer.length - 12_288) - if (cutPoint > 0) { - logsBuffer.delete(0, cutPoint + 1) - } - } - // Affichage plain text — pas de re-render HTML coûteux. - logsView.text = logsBuffer.toString() - logsScroll.post { logsScroll.fullScroll(android.widget.ScrollView.FOCUS_DOWN) } - } - - private fun renderState(st: PipelineState) { - when (st) { - is PipelineState.Idle -> { - statusView.text = "Status: Idle (prêt)" - pttButton.isEnabled = true - pttButton.text = "▶ Parler (VAD)" - } - is PipelineState.Listening -> { - statusView.text = "Status: Écoute… (parlez, VAD termine)" - pttButton.text = "■ Annuler" - pttButton.isEnabled = true - } - is PipelineState.Processing -> { - val phase = when (st.phase) { - PipelineState.Processing.Phase.Stt -> "STT…" - PipelineState.Processing.Phase.Thinker -> "Thinker (Phase 2)" - PipelineState.Processing.Phase.Speaker -> "Speaker (Phase 2)" - PipelineState.Processing.Phase.TtsStart -> "TTS init (Phase 3)" - } - statusView.text = "Status: $phase" - pttButton.isEnabled = false - bubbleKazeia.text = "..." - } - is PipelineState.Speaking -> { - statusView.text = "Status: Lecture audio (Phase 3)" - pttButton.isEnabled = false - } - is PipelineState.Error -> { - statusView.text = "ERREUR: ${st.msg}" - pttButton.isEnabled = true - } - } - } - - private fun handleEvent(ev: UiEvent) { - when (ev) { - is UiEvent.UserSpeechFinalized -> { - bubbleUser.text = ev.text.ifBlank { "(silence)" } - } - is UiEvent.KazeiaResponseReady -> { - bubbleKazeia.text = ev.text - } - is UiEvent.KazeiaSpeakingDone -> { - // Phase 3 : audio terminé - } - } - } - - private fun onPttClicked() { - val s = service ?: return - // Si on est déjà en écoute → annuler. - if (s.state.value is PipelineState.Listening) { - s.cancelListening() - return - } - if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { - permLauncher.launch(Manifest.permission.RECORD_AUDIO) - return - } - bubbleUser.text = "" - bubbleKazeia.text = "" - s.startVadListening() - } - - override fun onDestroy() { - super.onDestroy() - metricsSampler.stop() - service?.cancelListening() - if (bound) { - unbindService(connection) - bound = false - } - } -} diff --git a/kazeia-android/app/src/main/java/com/kazeia/v2/ExecutorchDaemon.kt b/kazeia-android/app/src/main/java/com/kazeia/v2/ExecutorchDaemon.kt deleted file mode 100644 index 72d8bb0..0000000 --- a/kazeia-android/app/src/main/java/com/kazeia/v2/ExecutorchDaemon.kt +++ /dev/null @@ -1,230 +0,0 @@ -package com.kazeia.v2 - -import java.io.BufferedReader -import java.io.File -import java.io.InputStreamReader -import java.io.OutputStreamWriter - -/** - * Wrapper Kotlin pour `qnn_llama_runner --daemon_mode` (cascade ExecuTorch). - * - * Reproduit fidèlement le protocole du PoC Python `cascade_executorch_daemon.py` : - * - * 1. `start()` lance le binaire avec `cwd = /data/local/tmp/kazeia-et`, - * `LD_LIBRARY_PATH=.`, `ADSP_LIBRARY_PATH=.`. Lit stdout jusqu'à voir - * `READY\n` (model load ~1.3 s pour 4B, ~4.5 s pour 8B). - * - * 2. `call(prompt, system, seqLen, temperature) -> Result` : - * - écrit prompt+system dans `outputs/_p_