feat(tts): adapter CosyVoice3 + dispatch ttsEngine="cosyvoice" (câblage, libs en attente)
Câblage Kotlin/APK du moteur TTS clonage vocal CosyVoice3 distillé livré par
kazeia-engine (dist/cosyvoice). Mirror du pattern sttEngine/ttsEngine="lib".
- CosyVoiceTtsEngine.kt : CosyVoiceJni (5 symboles) + adapter implémentant
com.kazeia.core.TtsEngine. Cache de voix (.cvps), synthèse phrase-par-phrase,
streaming AudioTrack MODE_STREAM en ENCODING_PCM_FLOAT @24 kHz (1er son joué
pendant que la phrase suivante se génère ; RTF~1.4 masqué par le streaming),
suspension jusqu'au drain réel (marker) pour garder le micro coupé (anti-écho).
- KazeiaService : branche when(ttsEngine) { "cosyvoice" -> ... } à l'init.
- ConfigStore : doc du nouveau backend.
⚠ NON validé device : BLOQUEUR côté libs engine. Les .so livrés embarquent leurs
propres libggml*.so (SONAME identiques au stack LLM/TTS gelé, 535 symboles ggml_*
en collision) → interposition ELF dans le namespace JNI (RTLD_GLOBAL) : le premier
libggml chargé (LLM, au boot) gagnerait, libcosyvoice se lierait au mauvais ggml →
crash. Ni écraser ni omettre les libggml app ne marche (consommés par libllama/
libkazeia_engine/libkazeia_tts gelés). Fix attendu : ggml statique + visibilité
hidden dans libcosyvoice.so côté engine. Code prêt à valider dès libs corrigées.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kazeia Team
parent
16d56bfd4d
commit
ed61737199
|
|
@ -48,6 +48,9 @@ class ConfigStore private constructor(private val file: File) {
|
|||
* libllama.so fork ql embarqué depuis 2026-06-02), "prod" = Qwen3TtsEngine
|
||||
* legacy — CRASHE (SIGSEGV nativeInit, dérive ABI llama_context_params) tant
|
||||
* que les .so legacy ne sont pas recompilés contre les headers fork ql.
|
||||
* "cosyvoice" = CosyVoiceTtsEngine (clonage vocal 9 langues, CPU-only) —
|
||||
* câblé mais NON activable tant que la collision libggml des libs livrées
|
||||
* n'est pas levée côté engine (cf CosyVoiceTtsEngine.kt).
|
||||
* Default "lib" depuis 2026-06-11 : l'ancien défaut "prod" mettait toute
|
||||
* install fraîche en crash-loop au démarrage du service. */
|
||||
val ttsEngine: String = "lib",
|
||||
|
|
|
|||
|
|
@ -753,24 +753,37 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
|
||||
val nativeLibDir = applicationInfo.nativeLibraryDir
|
||||
|
||||
// TTS legacy gated par ttsEngine="prod". Si "lib" ou autre :
|
||||
// skip l'init Qwen3TtsEngine (incompat ABI avec libllama fork-ql
|
||||
// déployé pour LLM engine). Step B câblera l'adapter lib.
|
||||
val ttsBackend = runtimeConfig.ttsEngine
|
||||
if (ttsBackend == "lib") {
|
||||
_loadingState.value = LoadingState(15, "TTS engine…")
|
||||
val libTts = com.kazeia.tts.KazeiaTtsEngine(this@KazeiaService) { msg -> log("[TTS] $msg") }
|
||||
libTts.load()
|
||||
if (!libTts.isLoaded()) throw IllegalStateException("KazeiaTtsEngine load failed")
|
||||
tts = libTts
|
||||
log("TTS: KazeiaTtsEngine loaded (libkazeia_tts CPU-only)")
|
||||
} else {
|
||||
_loadingState.value = LoadingState(15, "TTS Qwen3…")
|
||||
val qwenTts = com.kazeia.tts.Qwen3TtsEngine(nativeLibDir, this@KazeiaService) { msg -> log("[TTS] $msg") }
|
||||
qwenTts.load("$modelsDir/qwen3-tts-npu")
|
||||
if (!qwenTts.isLoaded()) throw IllegalStateException("Qwen3-TTS load failed (no fallback in frozen stack)")
|
||||
tts = qwenTts
|
||||
log("TTS: Qwen3-TTS loaded (Hexagon NPU + ggml-cpu)")
|
||||
// Dispatch backend TTS via ttsEngine. "prod" = Qwen3TtsEngine legacy
|
||||
// (incompat ABI avec libllama fork-ql -> SIGSEGV sur install fraîche,
|
||||
// cf feedback_tts_engine_default_lib). "lib" = KazeiaTtsEngine
|
||||
// (libkazeia_tts CPU). "cosyvoice" = CosyVoiceTtsEngine (clonage vocal
|
||||
// 9 langues, CPU-only) — NON validé device tant que la collision
|
||||
// libggml n'est pas levée côté libs engine (cf CosyVoiceTtsEngine).
|
||||
when (runtimeConfig.ttsEngine) {
|
||||
"cosyvoice" -> {
|
||||
_loadingState.value = LoadingState(15, "TTS CosyVoice…")
|
||||
val cv = com.kazeia.tts.CosyVoiceTtsEngine(log = { msg -> log("[TTS] $msg") })
|
||||
cv.load()
|
||||
if (!cv.isLoaded()) throw IllegalStateException("CosyVoice load failed")
|
||||
tts = cv
|
||||
log("TTS: CosyVoiceTtsEngine loaded (clonage vocal CPU-only)")
|
||||
}
|
||||
"lib" -> {
|
||||
_loadingState.value = LoadingState(15, "TTS engine…")
|
||||
val libTts = com.kazeia.tts.KazeiaTtsEngine(this@KazeiaService) { msg -> log("[TTS] $msg") }
|
||||
libTts.load()
|
||||
if (!libTts.isLoaded()) throw IllegalStateException("KazeiaTtsEngine load failed")
|
||||
tts = libTts
|
||||
log("TTS: KazeiaTtsEngine loaded (libkazeia_tts CPU-only)")
|
||||
}
|
||||
else -> {
|
||||
_loadingState.value = LoadingState(15, "TTS Qwen3…")
|
||||
val qwenTts = com.kazeia.tts.Qwen3TtsEngine(nativeLibDir, this@KazeiaService) { msg -> log("[TTS] $msg") }
|
||||
qwenTts.load("$modelsDir/qwen3-tts-npu")
|
||||
if (!qwenTts.isLoaded()) throw IllegalStateException("Qwen3-TTS load failed (no fallback in frozen stack)")
|
||||
tts = qwenTts
|
||||
log("TTS: Qwen3-TTS loaded (Hexagon NPU + ggml-cpu)")
|
||||
}
|
||||
}
|
||||
_loadingState.value = LoadingState(20, "TTS OK")
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,221 @@
|
|||
// Adapter com.kazeia.core.TtsEngine -> CosyVoice3 distillé (libkazeia_cosyvoice.so).
|
||||
// Clonage vocal zero-shot, 9 langues (fr,en,de,es,it,ru,zh,ja,ko), CPU-only (pas de
|
||||
// FastRPC/SELinux). Sélectionné quand ttsEngine="cosyvoice".
|
||||
//
|
||||
// Assets (sous KazeiaPaths.modelsDir) :
|
||||
// cv3_distilled_30k.gguf (~995 Mo, q8_0, flow distillé 2-NFE)
|
||||
// cosyvoice/<voiceId>.cvps (profil voix pré-enrôlé OFFLINE via cv_make_prompt.py)
|
||||
//
|
||||
// ⚠ BLOQUEUR LIVRAISON (2026-06-14) : les libs cosyvoice livrées embarquent leurs
|
||||
// propres libggml*.so (SONAME identiques à ceux du stack LLM/TTS gelé, 535 symboles
|
||||
// ggml_* en collision) -> interposition ELF dans le namespace JNI. En attente de libs
|
||||
// engine avec ggml statique+hidden dans libcosyvoice.so. Ce code est prêt mais NON
|
||||
// validé device tant que les libs corrigées ne sont pas intégrées. cf message dev.
|
||||
package com.kazeia.tts
|
||||
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioTrack
|
||||
import com.kazeia.core.TtsEngine
|
||||
import com.kazeia.core.TtsResult
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
/** Bridge JNI CosyVoice3 (5 symboles, livré par kazeia-engine/dist/cosyvoice). */
|
||||
object CosyVoiceJni {
|
||||
external fun nativeLoad(ggufPath: String, nfe: Int, nThreads: Int): Long
|
||||
external fun nativeLoadVoice(handle: Long, cvpsPath: String): Long
|
||||
external fun nativeSynthesize(voiceHandle: Long, text: String, speed: Float): FloatArray?
|
||||
external fun nativeFreeVoice(voiceHandle: Long)
|
||||
external fun nativeFree(handle: Long)
|
||||
|
||||
@Volatile private var loaded = false
|
||||
@Synchronized fun ensureLibrary() {
|
||||
if (!loaded) { System.loadLibrary("kazeia_cosyvoice"); loaded = true }
|
||||
}
|
||||
}
|
||||
|
||||
class CosyVoiceTtsEngine(
|
||||
private val log: (String) -> Unit = {},
|
||||
private val nfe: Int = 2, // pas de flow du student distillé (2 = qualité teacher)
|
||||
private val nThreads: Int = 8, // V79 : 8 cœurs perf
|
||||
private val defaultVoiceId: String = DEFAULT_VOICE
|
||||
) : TtsEngine {
|
||||
|
||||
companion object {
|
||||
const val SR = 24_000
|
||||
const val DEFAULT_VOICE = "damien"
|
||||
// 9 langues couvertes par CosyVoice3. Hors de cette liste -> Chatterbox (à venir).
|
||||
val LANGS = setOf("fr", "en", "de", "es", "it", "ru", "zh", "ja", "ko")
|
||||
private const val MODEL = "cv3_distilled_30k.gguf"
|
||||
}
|
||||
|
||||
private var handle: Long = 0L
|
||||
// Cache des voix chargées (un .cvps -> un handle natif). Réutilisé entre tours.
|
||||
private val voices = HashMap<String, Long>()
|
||||
@Volatile private var currentVoiceId: String = defaultVoiceId
|
||||
@Volatile private var audioTrack: AudioTrack? = null
|
||||
|
||||
private fun base(): String = com.kazeia.KazeiaApplication.MODELS_DIR
|
||||
private fun cvpsPath(voiceId: String): String = "${base()}/cosyvoice/$voiceId.cvps"
|
||||
|
||||
override suspend fun load(modelPath: String?, voiceId: String?) {
|
||||
if (handle != 0L) return
|
||||
withContext(Dispatchers.Default) {
|
||||
CosyVoiceJni.ensureLibrary()
|
||||
val gguf = modelPath ?: "${base()}/$MODEL"
|
||||
require(File(gguf).exists()) { "CosyVoice: modèle absent: $gguf" }
|
||||
val t0 = System.currentTimeMillis()
|
||||
handle = CosyVoiceJni.nativeLoad(gguf, nfe, nThreads)
|
||||
require(handle != 0L) { "CosyVoice: échec chargement modèle $gguf" }
|
||||
log("load OK in ${System.currentTimeMillis() - t0}ms (nfe=$nfe, threads=$nThreads)")
|
||||
(voiceId ?: defaultVoiceId).let { ensureVoice(it) }
|
||||
}
|
||||
}
|
||||
|
||||
override fun isLoaded(): Boolean = handle != 0L
|
||||
|
||||
/** Charge (et met en cache) un profil voix. Devient la voix courante. */
|
||||
fun setVoice(voiceId: String) {
|
||||
currentVoiceId = voiceId
|
||||
if (handle != 0L) ensureVoice(voiceId)
|
||||
}
|
||||
|
||||
private fun ensureVoice(voiceId: String): Long {
|
||||
voices[voiceId]?.let { currentVoiceId = voiceId; return it }
|
||||
val path = cvpsPath(voiceId)
|
||||
if (!File(path).exists()) { log("voix absente: $path — voix inchangée ($currentVoiceId)"); return 0L }
|
||||
val v = CosyVoiceJni.nativeLoadVoice(handle, path)
|
||||
if (v == 0L) { log("échec chargement voix $path"); return 0L }
|
||||
voices[voiceId] = v
|
||||
currentVoiceId = voiceId
|
||||
log("voix '$voiceId' chargée")
|
||||
return v
|
||||
}
|
||||
|
||||
/** Découpe en phrases (le streaming joue chaque phrase dès qu'elle est synthétisée). */
|
||||
private fun sentences(text: String): List<String> =
|
||||
Regex("[^.!?…]+[.!?…]*").findAll(text)
|
||||
.map { it.value.trim() }.filter { it.isNotEmpty() }.toList()
|
||||
.ifEmpty { listOf(text.trim()).filter { it.isNotEmpty() } }
|
||||
|
||||
private fun voiceHandleOrThrow(): Long {
|
||||
require(handle != 0L) { "CosyVoice: not loaded" }
|
||||
val v = voices[currentVoiceId] ?: ensureVoice(currentVoiceId)
|
||||
require(v != 0L) { "CosyVoice: aucune voix chargée ('$currentVoiceId')" }
|
||||
return v
|
||||
}
|
||||
|
||||
override suspend fun synthesize(text: String, language: String): TtsResult =
|
||||
withContext(Dispatchers.Default) {
|
||||
val v = voiceHandleOrThrow()
|
||||
val out = ArrayList<Float>()
|
||||
for (s in sentences(text)) {
|
||||
val pcm = CosyVoiceJni.nativeSynthesize(v, s, 1.0f) ?: continue
|
||||
pcm.forEach { out.add(it) }
|
||||
}
|
||||
val shorts = ShortArray(out.size) { floatToPcm16(out[it]) }
|
||||
TtsResult(audioData = shorts, sampleRate = SR, durationMs = shorts.size * 1000L / SR)
|
||||
}
|
||||
|
||||
override suspend fun synthesizeAndPlay(
|
||||
text: String,
|
||||
language: String,
|
||||
onStart: (() -> Unit)?,
|
||||
onComplete: (() -> Unit)?
|
||||
) {
|
||||
val v = voiceHandleOrThrow()
|
||||
val segs = sentences(text)
|
||||
if (segs.isEmpty()) { onComplete?.invoke(); return }
|
||||
|
||||
// AudioTrack flottant 24 kHz mono en MODE_STREAM : on commence à jouer dès
|
||||
// la 1ʳᵉ phrase synthétisée pendant que la suivante se génère (RTF>1 -> le
|
||||
// streaming masque la latence). Suspend jusqu'à la FIN réelle (marker) pour
|
||||
// que le micro reste coupé pendant toute la lecture (anti-écho, cf KazeiaTts).
|
||||
val minBuf = AudioTrack.getMinBufferSize(
|
||||
SR, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_FLOAT
|
||||
).coerceAtLeast(SR * 4) // ~2 s de marge mono float
|
||||
val track = AudioTrack.Builder()
|
||||
.setAudioAttributes(AudioAttributes.Builder()
|
||||
.setUsage(AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build())
|
||||
.setAudioFormat(AudioFormat.Builder()
|
||||
.setSampleRate(SR)
|
||||
.setEncoding(AudioFormat.ENCODING_PCM_FLOAT)
|
||||
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
|
||||
.build())
|
||||
.setBufferSizeInBytes(minBuf)
|
||||
.setTransferMode(AudioTrack.MODE_STREAM)
|
||||
.build()
|
||||
audioTrack = track
|
||||
|
||||
var totalFrames = 0
|
||||
var started = false
|
||||
val t0 = System.currentTimeMillis()
|
||||
try {
|
||||
withContext(Dispatchers.Default) {
|
||||
for ((i, s) in segs.withIndex()) {
|
||||
val pcm = CosyVoiceJni.nativeSynthesize(v, s, 1.0f)
|
||||
if (pcm == null || pcm.isEmpty()) continue
|
||||
if (!started) {
|
||||
started = true
|
||||
onStart?.invoke()
|
||||
track.play()
|
||||
log("1er son à ${System.currentTimeMillis() - t0}ms (phrase 1/${segs.size})")
|
||||
}
|
||||
// write bloquant (MODE_STREAM) : régule le débit sur la lecture.
|
||||
var off = 0
|
||||
while (off < pcm.size) {
|
||||
val n = track.write(pcm, off, pcm.size - off, AudioTrack.WRITE_BLOCKING)
|
||||
if (n < 0) { log("AudioTrack.write err=$n"); break }
|
||||
off += n
|
||||
}
|
||||
totalFrames += pcm.size
|
||||
}
|
||||
}
|
||||
} catch (e: Throwable) {
|
||||
log("synthesizeAndPlay erreur: ${e.message}")
|
||||
}
|
||||
|
||||
if (!started || totalFrames == 0) { stopInternal(track); onComplete?.invoke(); return }
|
||||
|
||||
// Attend le drain réel des échantillons écrits avant de rendre la main.
|
||||
suspendCancellableCoroutine<Unit> { cont ->
|
||||
cont.invokeOnCancellation { stopInternal(track) }
|
||||
track.setNotificationMarkerPosition(totalFrames)
|
||||
track.setPlaybackPositionUpdateListener(object : AudioTrack.OnPlaybackPositionUpdateListener {
|
||||
override fun onMarkerReached(t: AudioTrack?) { if (cont.isActive) cont.resume(Unit) }
|
||||
override fun onPeriodicNotification(t: AudioTrack?) {}
|
||||
})
|
||||
// MODE_STREAM : stop() laisse drainer ce qui est déjà écrit puis atteint le marker.
|
||||
try { track.stop() } catch (_: Exception) {}
|
||||
}
|
||||
stopInternal(track)
|
||||
onComplete?.invoke()
|
||||
}
|
||||
|
||||
override fun stop() { audioTrack?.let { stopInternal(it) } }
|
||||
|
||||
private fun stopInternal(track: AudioTrack) {
|
||||
try { track.pause(); track.flush() } catch (_: Exception) {}
|
||||
try { track.stop() } catch (_: Exception) {}
|
||||
try { track.release() } catch (_: Exception) {}
|
||||
if (audioTrack === track) audioTrack = null
|
||||
}
|
||||
|
||||
override fun release() {
|
||||
audioTrack?.let { stopInternal(it) }
|
||||
voices.values.forEach { try { CosyVoiceJni.nativeFreeVoice(it) } catch (_: Exception) {} }
|
||||
voices.clear()
|
||||
if (handle != 0L) { try { CosyVoiceJni.nativeFree(handle) } catch (_: Exception) {}; handle = 0L }
|
||||
}
|
||||
|
||||
private fun floatToPcm16(f: Float): Short {
|
||||
val c = if (f > 1f) 1f else if (f < -1f) -1f else f
|
||||
return (c * 32767f).toInt().toShort()
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue