From ed61737199d3fbe9b27141cb4cd640277ea4e347 Mon Sep 17 00:00:00 2001 From: Kazeia Team Date: Sun, 14 Jun 2026 17:53:30 +0200 Subject: [PATCH] =?UTF-8?q?feat(tts):=20adapter=20CosyVoice3=20+=20dispatc?= =?UTF-8?q?h=20ttsEngine=3D"cosyvoice"=20(c=C3=A2blage,=20libs=20en=20atte?= =?UTF-8?q?nte)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Câblage Kotlin/APK du moteur TTS clonage vocal CosyVoice3 distillé livré par kazeia-engine (dist/cosyvoice). Mirror du pattern sttEngine/ttsEngine="lib". - CosyVoiceTtsEngine.kt : CosyVoiceJni (5 symboles) + adapter implémentant com.kazeia.core.TtsEngine. Cache de voix (.cvps), synthèse phrase-par-phrase, streaming AudioTrack MODE_STREAM en ENCODING_PCM_FLOAT @24 kHz (1er son joué pendant que la phrase suivante se génère ; RTF~1.4 masqué par le streaming), suspension jusqu'au drain réel (marker) pour garder le micro coupé (anti-écho). - KazeiaService : branche when(ttsEngine) { "cosyvoice" -> ... } à l'init. - ConfigStore : doc du nouveau backend. ⚠ NON validé device : BLOQUEUR côté libs engine. Les .so livrés embarquent leurs propres libggml*.so (SONAME identiques au stack LLM/TTS gelé, 535 symboles ggml_* en collision) → interposition ELF dans le namespace JNI (RTLD_GLOBAL) : le premier libggml chargé (LLM, au boot) gagnerait, libcosyvoice se lierait au mauvais ggml → crash. Ni écraser ni omettre les libggml app ne marche (consommés par libllama/ libkazeia_engine/libkazeia_tts gelés). Fix attendu : ggml statique + visibilité hidden dans libcosyvoice.so côté engine. Code prêt à valider dès libs corrigées. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../java/com/kazeia/config/ConfigStore.kt | 3 + .../java/com/kazeia/service/KazeiaService.kt | 49 ++-- .../java/com/kazeia/tts/CosyVoiceTtsEngine.kt | 221 ++++++++++++++++++ 3 files changed, 255 insertions(+), 18 deletions(-) create mode 100644 kazeia-android/app/src/main/java/com/kazeia/tts/CosyVoiceTtsEngine.kt diff --git a/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt b/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt index e72c60a..38fe411 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/config/ConfigStore.kt @@ -48,6 +48,9 @@ class ConfigStore private constructor(private val file: File) { * libllama.so fork ql embarqué depuis 2026-06-02), "prod" = Qwen3TtsEngine * legacy — CRASHE (SIGSEGV nativeInit, dérive ABI llama_context_params) tant * que les .so legacy ne sont pas recompilés contre les headers fork ql. + * "cosyvoice" = CosyVoiceTtsEngine (clonage vocal 9 langues, CPU-only) — + * câblé mais NON activable tant que la collision libggml des libs livrées + * n'est pas levée côté engine (cf CosyVoiceTtsEngine.kt). * Default "lib" depuis 2026-06-11 : l'ancien défaut "prod" mettait toute * install fraîche en crash-loop au démarrage du service. */ val ttsEngine: String = "lib", diff --git a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt index 8b1e8f7..84ca489 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt @@ -753,24 +753,37 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think val nativeLibDir = applicationInfo.nativeLibraryDir - // TTS legacy gated par ttsEngine="prod". Si "lib" ou autre : - // skip l'init Qwen3TtsEngine (incompat ABI avec libllama fork-ql - // déployé pour LLM engine). Step B câblera l'adapter lib. - val ttsBackend = runtimeConfig.ttsEngine - if (ttsBackend == "lib") { - _loadingState.value = LoadingState(15, "TTS engine…") - val libTts = com.kazeia.tts.KazeiaTtsEngine(this@KazeiaService) { msg -> log("[TTS] $msg") } - libTts.load() - if (!libTts.isLoaded()) throw IllegalStateException("KazeiaTtsEngine load failed") - tts = libTts - log("TTS: KazeiaTtsEngine loaded (libkazeia_tts CPU-only)") - } else { - _loadingState.value = LoadingState(15, "TTS Qwen3…") - val qwenTts = com.kazeia.tts.Qwen3TtsEngine(nativeLibDir, this@KazeiaService) { msg -> log("[TTS] $msg") } - qwenTts.load("$modelsDir/qwen3-tts-npu") - if (!qwenTts.isLoaded()) throw IllegalStateException("Qwen3-TTS load failed (no fallback in frozen stack)") - tts = qwenTts - log("TTS: Qwen3-TTS loaded (Hexagon NPU + ggml-cpu)") + // Dispatch backend TTS via ttsEngine. "prod" = Qwen3TtsEngine legacy + // (incompat ABI avec libllama fork-ql -> SIGSEGV sur install fraîche, + // cf feedback_tts_engine_default_lib). "lib" = KazeiaTtsEngine + // (libkazeia_tts CPU). "cosyvoice" = CosyVoiceTtsEngine (clonage vocal + // 9 langues, CPU-only) — NON validé device tant que la collision + // libggml n'est pas levée côté libs engine (cf CosyVoiceTtsEngine). + when (runtimeConfig.ttsEngine) { + "cosyvoice" -> { + _loadingState.value = LoadingState(15, "TTS CosyVoice…") + val cv = com.kazeia.tts.CosyVoiceTtsEngine(log = { msg -> log("[TTS] $msg") }) + cv.load() + if (!cv.isLoaded()) throw IllegalStateException("CosyVoice load failed") + tts = cv + log("TTS: CosyVoiceTtsEngine loaded (clonage vocal CPU-only)") + } + "lib" -> { + _loadingState.value = LoadingState(15, "TTS engine…") + val libTts = com.kazeia.tts.KazeiaTtsEngine(this@KazeiaService) { msg -> log("[TTS] $msg") } + libTts.load() + if (!libTts.isLoaded()) throw IllegalStateException("KazeiaTtsEngine load failed") + tts = libTts + log("TTS: KazeiaTtsEngine loaded (libkazeia_tts CPU-only)") + } + else -> { + _loadingState.value = LoadingState(15, "TTS Qwen3…") + val qwenTts = com.kazeia.tts.Qwen3TtsEngine(nativeLibDir, this@KazeiaService) { msg -> log("[TTS] $msg") } + qwenTts.load("$modelsDir/qwen3-tts-npu") + if (!qwenTts.isLoaded()) throw IllegalStateException("Qwen3-TTS load failed (no fallback in frozen stack)") + tts = qwenTts + log("TTS: Qwen3-TTS loaded (Hexagon NPU + ggml-cpu)") + } } _loadingState.value = LoadingState(20, "TTS OK") diff --git a/kazeia-android/app/src/main/java/com/kazeia/tts/CosyVoiceTtsEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/tts/CosyVoiceTtsEngine.kt new file mode 100644 index 0000000..40183d7 --- /dev/null +++ b/kazeia-android/app/src/main/java/com/kazeia/tts/CosyVoiceTtsEngine.kt @@ -0,0 +1,221 @@ +// Adapter com.kazeia.core.TtsEngine -> CosyVoice3 distillé (libkazeia_cosyvoice.so). +// Clonage vocal zero-shot, 9 langues (fr,en,de,es,it,ru,zh,ja,ko), CPU-only (pas de +// FastRPC/SELinux). Sélectionné quand ttsEngine="cosyvoice". +// +// Assets (sous KazeiaPaths.modelsDir) : +// cv3_distilled_30k.gguf (~995 Mo, q8_0, flow distillé 2-NFE) +// cosyvoice/.cvps (profil voix pré-enrôlé OFFLINE via cv_make_prompt.py) +// +// ⚠ BLOQUEUR LIVRAISON (2026-06-14) : les libs cosyvoice livrées embarquent leurs +// propres libggml*.so (SONAME identiques à ceux du stack LLM/TTS gelé, 535 symboles +// ggml_* en collision) -> interposition ELF dans le namespace JNI. En attente de libs +// engine avec ggml statique+hidden dans libcosyvoice.so. Ce code est prêt mais NON +// validé device tant que les libs corrigées ne sont pas intégrées. cf message dev. +package com.kazeia.tts + +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioTrack +import com.kazeia.core.TtsEngine +import com.kazeia.core.TtsResult +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import java.io.File +import kotlin.coroutines.resume + +/** Bridge JNI CosyVoice3 (5 symboles, livré par kazeia-engine/dist/cosyvoice). */ +object CosyVoiceJni { + external fun nativeLoad(ggufPath: String, nfe: Int, nThreads: Int): Long + external fun nativeLoadVoice(handle: Long, cvpsPath: String): Long + external fun nativeSynthesize(voiceHandle: Long, text: String, speed: Float): FloatArray? + external fun nativeFreeVoice(voiceHandle: Long) + external fun nativeFree(handle: Long) + + @Volatile private var loaded = false + @Synchronized fun ensureLibrary() { + if (!loaded) { System.loadLibrary("kazeia_cosyvoice"); loaded = true } + } +} + +class CosyVoiceTtsEngine( + private val log: (String) -> Unit = {}, + private val nfe: Int = 2, // pas de flow du student distillé (2 = qualité teacher) + private val nThreads: Int = 8, // V79 : 8 cœurs perf + private val defaultVoiceId: String = DEFAULT_VOICE +) : TtsEngine { + + companion object { + const val SR = 24_000 + const val DEFAULT_VOICE = "damien" + // 9 langues couvertes par CosyVoice3. Hors de cette liste -> Chatterbox (à venir). + val LANGS = setOf("fr", "en", "de", "es", "it", "ru", "zh", "ja", "ko") + private const val MODEL = "cv3_distilled_30k.gguf" + } + + private var handle: Long = 0L + // Cache des voix chargées (un .cvps -> un handle natif). Réutilisé entre tours. + private val voices = HashMap() + @Volatile private var currentVoiceId: String = defaultVoiceId + @Volatile private var audioTrack: AudioTrack? = null + + private fun base(): String = com.kazeia.KazeiaApplication.MODELS_DIR + private fun cvpsPath(voiceId: String): String = "${base()}/cosyvoice/$voiceId.cvps" + + override suspend fun load(modelPath: String?, voiceId: String?) { + if (handle != 0L) return + withContext(Dispatchers.Default) { + CosyVoiceJni.ensureLibrary() + val gguf = modelPath ?: "${base()}/$MODEL" + require(File(gguf).exists()) { "CosyVoice: modèle absent: $gguf" } + val t0 = System.currentTimeMillis() + handle = CosyVoiceJni.nativeLoad(gguf, nfe, nThreads) + require(handle != 0L) { "CosyVoice: échec chargement modèle $gguf" } + log("load OK in ${System.currentTimeMillis() - t0}ms (nfe=$nfe, threads=$nThreads)") + (voiceId ?: defaultVoiceId).let { ensureVoice(it) } + } + } + + override fun isLoaded(): Boolean = handle != 0L + + /** Charge (et met en cache) un profil voix. Devient la voix courante. */ + fun setVoice(voiceId: String) { + currentVoiceId = voiceId + if (handle != 0L) ensureVoice(voiceId) + } + + private fun ensureVoice(voiceId: String): Long { + voices[voiceId]?.let { currentVoiceId = voiceId; return it } + val path = cvpsPath(voiceId) + if (!File(path).exists()) { log("voix absente: $path — voix inchangée ($currentVoiceId)"); return 0L } + val v = CosyVoiceJni.nativeLoadVoice(handle, path) + if (v == 0L) { log("échec chargement voix $path"); return 0L } + voices[voiceId] = v + currentVoiceId = voiceId + log("voix '$voiceId' chargée") + return v + } + + /** Découpe en phrases (le streaming joue chaque phrase dès qu'elle est synthétisée). */ + private fun sentences(text: String): List = + Regex("[^.!?…]+[.!?…]*").findAll(text) + .map { it.value.trim() }.filter { it.isNotEmpty() }.toList() + .ifEmpty { listOf(text.trim()).filter { it.isNotEmpty() } } + + private fun voiceHandleOrThrow(): Long { + require(handle != 0L) { "CosyVoice: not loaded" } + val v = voices[currentVoiceId] ?: ensureVoice(currentVoiceId) + require(v != 0L) { "CosyVoice: aucune voix chargée ('$currentVoiceId')" } + return v + } + + override suspend fun synthesize(text: String, language: String): TtsResult = + withContext(Dispatchers.Default) { + val v = voiceHandleOrThrow() + val out = ArrayList() + for (s in sentences(text)) { + val pcm = CosyVoiceJni.nativeSynthesize(v, s, 1.0f) ?: continue + pcm.forEach { out.add(it) } + } + val shorts = ShortArray(out.size) { floatToPcm16(out[it]) } + TtsResult(audioData = shorts, sampleRate = SR, durationMs = shorts.size * 1000L / SR) + } + + override suspend fun synthesizeAndPlay( + text: String, + language: String, + onStart: (() -> Unit)?, + onComplete: (() -> Unit)? + ) { + val v = voiceHandleOrThrow() + val segs = sentences(text) + if (segs.isEmpty()) { onComplete?.invoke(); return } + + // AudioTrack flottant 24 kHz mono en MODE_STREAM : on commence à jouer dès + // la 1ʳᵉ phrase synthétisée pendant que la suivante se génère (RTF>1 -> le + // streaming masque la latence). Suspend jusqu'à la FIN réelle (marker) pour + // que le micro reste coupé pendant toute la lecture (anti-écho, cf KazeiaTts). + val minBuf = AudioTrack.getMinBufferSize( + SR, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_FLOAT + ).coerceAtLeast(SR * 4) // ~2 s de marge mono float + val track = AudioTrack.Builder() + .setAudioAttributes(AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build()) + .setAudioFormat(AudioFormat.Builder() + .setSampleRate(SR) + .setEncoding(AudioFormat.ENCODING_PCM_FLOAT) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .build()) + .setBufferSizeInBytes(minBuf) + .setTransferMode(AudioTrack.MODE_STREAM) + .build() + audioTrack = track + + var totalFrames = 0 + var started = false + val t0 = System.currentTimeMillis() + try { + withContext(Dispatchers.Default) { + for ((i, s) in segs.withIndex()) { + val pcm = CosyVoiceJni.nativeSynthesize(v, s, 1.0f) + if (pcm == null || pcm.isEmpty()) continue + if (!started) { + started = true + onStart?.invoke() + track.play() + log("1er son à ${System.currentTimeMillis() - t0}ms (phrase 1/${segs.size})") + } + // write bloquant (MODE_STREAM) : régule le débit sur la lecture. + var off = 0 + while (off < pcm.size) { + val n = track.write(pcm, off, pcm.size - off, AudioTrack.WRITE_BLOCKING) + if (n < 0) { log("AudioTrack.write err=$n"); break } + off += n + } + totalFrames += pcm.size + } + } + } catch (e: Throwable) { + log("synthesizeAndPlay erreur: ${e.message}") + } + + if (!started || totalFrames == 0) { stopInternal(track); onComplete?.invoke(); return } + + // Attend le drain réel des échantillons écrits avant de rendre la main. + suspendCancellableCoroutine { cont -> + cont.invokeOnCancellation { stopInternal(track) } + track.setNotificationMarkerPosition(totalFrames) + track.setPlaybackPositionUpdateListener(object : AudioTrack.OnPlaybackPositionUpdateListener { + override fun onMarkerReached(t: AudioTrack?) { if (cont.isActive) cont.resume(Unit) } + override fun onPeriodicNotification(t: AudioTrack?) {} + }) + // MODE_STREAM : stop() laisse drainer ce qui est déjà écrit puis atteint le marker. + try { track.stop() } catch (_: Exception) {} + } + stopInternal(track) + onComplete?.invoke() + } + + override fun stop() { audioTrack?.let { stopInternal(it) } } + + private fun stopInternal(track: AudioTrack) { + try { track.pause(); track.flush() } catch (_: Exception) {} + try { track.stop() } catch (_: Exception) {} + try { track.release() } catch (_: Exception) {} + if (audioTrack === track) audioTrack = null + } + + override fun release() { + audioTrack?.let { stopInternal(it) } + voices.values.forEach { try { CosyVoiceJni.nativeFreeVoice(it) } catch (_: Exception) {} } + voices.clear() + if (handle != 0L) { try { CosyVoiceJni.nativeFree(handle) } catch (_: Exception) {}; handle = 0L } + } + + private fun floatToPcm16(f: Float): Short { + val c = if (f > 1f) 1f else if (f < -1f) -1f else f + return (c * 32767f).toInt().toShort() + } +}