chantier B engine #3 : TTS lib audible + E2E in-app validé
Trois fixes pour que le pipeline engine TTS tourne dans l'app réelle : - cp_inference: cp_keep_f16() retourne false par défaut. Garder les poids matmul CP en F16 faisait overflow les massive activations Qwen3 (>65504) -> NaN -> codes 0 -> audio inaudible. F32 = ref cp_runner bit-exact. KZTTS_CP_KEEP_F16=1 réactive l'ancien path (debug perf). - tts_engine: arming EOS naturel gated derrière KZTTS_EOS_NATURAL (off par défaut). C'était un workaround de l'ère NaN qui tronquait la fin de phrase (frames 33/37). Depuis le fix F32 le talker se termine seul (cb0==codec_eos), frames 40/42. Masking + anti-repeat + hard-fallback conservés. - kazeia_tts_jni: freopen stderr gated derrière KZTTS_STDERR_LOG (plus de redirection /sdcard par défaut en prod). - build_kazeia_tts.sh: speaker_encoder.cpp + kazeia_mel.cpp ajoutés aux SRCS. Validé E2E in-app : LLM lib (q35-lmq4 GGUF) + TTS lib coexistent sans collision libllama, réponse FR audible jouée via AudioTrack 24kHz. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Richard Loyer
parent
e55cd175c8
commit
f4c6e1290c
|
|
@ -18,6 +18,8 @@ CFLAGS=( -std=c++17 -O3 -fPIC
|
|||
SRCS=(
|
||||
"$JNI/kazeia_tts_jni.cpp"
|
||||
"$JNI/tts_engine.cpp"
|
||||
"$JNI/speaker_encoder.cpp"
|
||||
"$JNI/kazeia_mel.cpp"
|
||||
"$JNI/cp_inference.cpp"
|
||||
"$JNI/sampler.cpp"
|
||||
"$JNI/kazeia_text_tokenizer.cpp"
|
||||
|
|
|
|||
|
|
@ -43,8 +43,13 @@ static std::vector<float> read_floats(const char* path, size_t expect) {
|
|||
// éléments * 5 layers * 6 norms = 30 KB de norms, négligeable. Les matmul valent
|
||||
// ~5 * (3*1024*1024 attn + 3*1024*MLP ffn) qui dominent.
|
||||
static bool cp_keep_f16(const std::string& name) {
|
||||
// Tout ce qui n'est pas "_norm.weight" peut rester F16. Sécurité : on n'autorise
|
||||
// que les noms connus du CP qwen3 (5L). Les autres tombent en F32 par défaut.
|
||||
// RÉGRESSION CORRIGÉE : garder les poids matmul en F16 fait tourner le mul_mat en F16
|
||||
// (ggml convertit l'activation f32 -> f16 pour le vec_dot). Les massive activations
|
||||
// Qwen3 (hidden ±83, intermédiaires >65504) overflow en f16 -> inf -> NaN -> CP sort
|
||||
// des scores NaN -> argmax=0 -> audio inaudible. cp_runner.cpp (référence bit-exact)
|
||||
// convertit TOUT en F32 pour cette raison. On fait pareil : F16 désactivé par défaut.
|
||||
// KZTTS_CP_KEEP_F16=1 réactive l'ancien comportement (debug perf uniquement).
|
||||
if (!(getenv("KZTTS_CP_KEEP_F16") && atoi(getenv("KZTTS_CP_KEEP_F16")) != 0)) return false;
|
||||
if (name.find("_norm.weight") != std::string::npos) return false;
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
#include "tts_engine.h"
|
||||
#include <jni.h>
|
||||
#include <string>
|
||||
#include <cstdio>
|
||||
|
||||
namespace {
|
||||
|
||||
|
|
@ -33,6 +34,12 @@ Java_com_kazeia_tts_TtsJni_nativeLoad(JNIEnv* env, jobject /*thiz*/,
|
|||
jstring dump_dir,
|
||||
jboolean use_htp, jint n_threads,
|
||||
jboolean cp_use_cache) {
|
||||
// DEBUG opt-in : redirige stderr natif vers un fichier app-lisible quand
|
||||
// KZTTS_STDERR_LOG est défini. En prod (env absent) on laisse stderr tel quel.
|
||||
if (const char * dbg = getenv("KZTTS_STDERR_LOG")) {
|
||||
freopen(dbg, "w", stderr);
|
||||
setvbuf(stderr, nullptr, _IONBF, 0);
|
||||
}
|
||||
JStr talker(env, talker_gguf);
|
||||
JStr vocab (env, vocab_gguf);
|
||||
JStr dump (env, dump_dir);
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ int main(int argc, char** argv) {
|
|||
|
||||
jlong h = p_load(env, self,
|
||||
(jstring)talker_gguf, (jstring)vocab_gguf, (jstring)dump_dir,
|
||||
/*useHtp=*/JNI_FALSE, /*nThreads=*/6, /*cpUseCache=*/JNI_TRUE);
|
||||
/*useHtp=*/JNI_FALSE, /*nThreads=*/6, /*cpUseCache=*/JNI_FALSE);
|
||||
if (h == 0) { fprintf(stderr, "nativeLoad returned 0\n"); dlclose(dl); return 4; }
|
||||
fprintf(stderr,"nativeLoad OK -> handle=%p\n", (void*)(uintptr_t)h);
|
||||
|
||||
|
|
|
|||
|
|
@ -245,8 +245,10 @@ TtsEngine * tts_engine_load(const TtsEngineLoadCfg & cfg) {
|
|||
cp.n_ctx = 1024;
|
||||
cp.n_batch = 1024;
|
||||
cp.n_threads = cfg.n_threads;
|
||||
cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
|
||||
cp.flash_attn_type = (getenv("KZTTS_TALKER_FA") && atoi(getenv("KZTTS_TALKER_FA")) != 0)
|
||||
? LLAMA_FLASH_ATTN_TYPE_ENABLED : LLAMA_FLASH_ATTN_TYPE_DISABLED;
|
||||
cp.embeddings = true;
|
||||
cp.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
||||
eng->talker_ctx = llama_init_from_model(eng->talker_m, cp);
|
||||
if (!eng->talker_ctx) { fprintf(stderr, "talker ctx FAIL\n"); delete eng; return nullptr; }
|
||||
eng->n_embd = llama_model_n_embd(eng->talker_m);
|
||||
|
|
@ -345,7 +347,14 @@ TtsEngine * tts_engine_load(const TtsEngineLoadCfg & cfg) {
|
|||
eng->decoder.owns_backends = true;
|
||||
dec_ok = eng->decoder.load_with_backends((D + "qwen3tts_decoder.gguf").c_str(), dec_backends);
|
||||
} else {
|
||||
dec_ok = eng->decoder.load((D + "qwen3tts_decoder.gguf").c_str());
|
||||
// CPU-only : sched sur backend CPU -> stage_bigvgan_sched (gallocr, peak mem bas)
|
||||
// au lieu du legacy stage_bigvgan (malloc 12 GB -> ggml_aligned_malloc NULL -> crash in-app).
|
||||
ggml_backend_t cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
|
||||
if (!cpu) { fprintf(stderr, "decoder: CPU backend init FAIL\n"); delete eng; return nullptr; }
|
||||
std::vector<ggml_backend_t> dec_backends{cpu};
|
||||
eng->decoder.owns_backends = true;
|
||||
setenv("KZTTS_DECODER_SCHED", "1", 1);
|
||||
dec_ok = eng->decoder.load_with_backends((D + "qwen3tts_decoder.gguf").c_str(), dec_backends);
|
||||
}
|
||||
if (!dec_ok) {
|
||||
fprintf(stderr, "decoder load FAIL\n"); delete eng; return nullptr;
|
||||
|
|
@ -550,7 +559,55 @@ TtsSynthesizeResult tts_engine_synthesize(TtsEngine * eng, const TtsSynthesizeCf
|
|||
const float* lp = llama_get_logits_ith(eng->talker_ctx, -1);
|
||||
memcpy(logits_buf.data(), lp, n_vocab * sizeof(float));
|
||||
}
|
||||
|
||||
// EOS-rank boost dynamique (port du legacy Qwen3TtsEngine.runInterleavedCpuFromEmbeds).
|
||||
// Sans ça le talker n'émet quasi jamais codec_eos et tourne jusqu'à max_steps :
|
||||
// audio fixe ~20 s + synth ~5x trop longue. On masque les tokens non-codebook
|
||||
// (sauf eos), puis dès 2/3 de la durée attendue on amplifie eos quand son rang
|
||||
// devient bas (le modèle « pense à terminer »). Désactivable via KZTTS_EOS_BOOST=0.
|
||||
const bool eos_boost_on = !(getenv("KZTTS_EOS_BOOST") && atoi(getenv("KZTTS_EOS_BOOST")) == 0);
|
||||
// Arming naturel (rang<trig) : workaround de l'ère NaN où le talker n'émettait jamais
|
||||
// codec_eos. Depuis le fix CP F32, le talker se termine seul correctement (cb0==eos),
|
||||
// et cet arming ne fait que TRONQUER la fin de phrase. OFF par défaut ; seul le
|
||||
// hard-fallback reste actif comme garde anti-runaway. KZTTS_EOS_NATURAL=1 le réactive.
|
||||
const bool eos_natural_on = (getenv("KZTTS_EOS_NATURAL") && atoi(getenv("KZTTS_EOS_NATURAL")) != 0);
|
||||
const int CB0_VOCAB = 2048; // codes audio valides [0, 2048)
|
||||
const int expected_steps = Nt * 4;
|
||||
const int eos_min_step = (expected_steps * 2) / 3;
|
||||
const int eos_hard_step = eos_min_step * 3;
|
||||
const int eos_rank_trig = 300;
|
||||
const float eos_boost_scl = 4.0f;
|
||||
const float NEG = -1e30f;
|
||||
int boost_active = 0, consec_low = 0;
|
||||
// Anti-repeat « seen » (port legacy l.1866) : pénalise les cb0 déjà émis pour
|
||||
// faire remonter le rang de codec_eos. Sans ça le rang reste >300 et l'arming
|
||||
// naturel ne se déclenche jamais -> on retombait sur le hard fallback brutal.
|
||||
std::vector<char> seen(n_vocab, 0);
|
||||
auto apply_eos = [&](float* lg, int gen_step) -> bool {
|
||||
for (int j = CB0_VOCAB; j < n_vocab; ++j) if (j != eng->codec_eos) lg[j] = NEG;
|
||||
for (int j = 0; j < n_vocab; ++j) if (seen[j]) lg[j] = (lg[j] > 0) ? lg[j] / 1.05f : lg[j] * 1.05f;
|
||||
if (!eos_boost_on) return false;
|
||||
const float eos_l = lg[eng->codec_eos];
|
||||
int rank = 0; for (int j = 0; j < n_vocab; ++j) if (lg[j] > eos_l) rank++;
|
||||
if (boost_active == 0 && gen_step >= eos_min_step) {
|
||||
if (eos_natural_on && rank < eos_rank_trig) {
|
||||
if (++consec_low >= 3) { boost_active = 1;
|
||||
fprintf(stderr, "eos: armed NATURAL at step %d (rank=%d)\n", gen_step, rank); }
|
||||
} else { consec_low = 0; }
|
||||
if (boost_active == 0 && gen_step >= eos_hard_step) { boost_active = 1;
|
||||
fprintf(stderr, "eos: armed HARD-FALLBACK at step %d (rank=%d)\n", gen_step, rank); }
|
||||
} else if (boost_active > 0) {
|
||||
boost_active++;
|
||||
}
|
||||
if (boost_active > 0) lg[eng->codec_eos] += boost_active * eos_boost_scl;
|
||||
int am = 0; float amv = lg[0];
|
||||
for (int j = 1; j < n_vocab; ++j) if (lg[j] > amv) { amv = lg[j]; am = j; }
|
||||
return am == eng->codec_eos; // eos boosté en tête -> stop immédiat
|
||||
};
|
||||
|
||||
apply_eos(logits_buf.data(), 0);
|
||||
int cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab);
|
||||
int last_cb0 = -1, cb0_run = 0;
|
||||
std::vector<float> hidden_for_cp(n_embd);
|
||||
{ const float* hh = llama_get_embeddings_ith(eng->talker_ctx, -1);
|
||||
if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float)); }
|
||||
|
|
@ -595,10 +652,19 @@ TtsSynthesizeResult tts_engine_synthesize(TtsEngine * eng, const TtsSynthesizeCf
|
|||
|
||||
for (int s = 0; s < cfg.max_steps; ++s) {
|
||||
if (cb0 == eng->codec_eos) break;
|
||||
// Garde anti-dégénérescence : même cb0 répété 9x => boucle morte, on stoppe.
|
||||
if (cb0 == last_cb0) { if (++cb0_run >= 9) { fprintf(stderr, "eos: degeneration break at step %d\n", s); break; } }
|
||||
else { last_cb0 = cb0; cb0_run = 1; }
|
||||
seen[cb0] = 1; // mémorise le cb0 courant pour la pénalité anti-repeat
|
||||
codes_engine.push_back(cb0);
|
||||
|
||||
const double tcp0 = now_s();
|
||||
const float* cb0_emb = eng->tok_embd.data() + (size_t)cb0 * n_embd;
|
||||
if (getenv("KZTTS_DBG_CP") && s < 4) {
|
||||
double hn = 0, en = 0; for (int d = 0; d < n_embd; ++d) { hn += hidden_for_cp[d]*hidden_for_cp[d]; en += cb0_emb[d]*cb0_emb[d]; }
|
||||
fprintf(stderr, "dbg_cp s=%d cb0=%d |hidden|=%.4f |cb0_emb|=%.4f h[0..3]=%.3f,%.3f,%.3f\n",
|
||||
s, cb0, sqrt(hn), sqrt(en), hidden_for_cp[0], hidden_for_cp[1], hidden_for_cp[2]);
|
||||
}
|
||||
int32_t cb15[15];
|
||||
if (eng->cp_use_cache) cp_predict_cached(eng->cp_state, hidden_for_cp.data(), cb0_emb, cb15);
|
||||
else cp_predict (eng->cp_state, hidden_for_cp.data(), cb0_emb, cb15);
|
||||
|
|
@ -632,7 +698,9 @@ TtsSynthesizeResult tts_engine_synthesize(TtsEngine * eng, const TtsSynthesizeCf
|
|||
const float* lp = llama_get_logits_ith(eng->talker_ctx, -1);
|
||||
memcpy(logits_buf.data(), lp, n_vocab * sizeof(float));
|
||||
}
|
||||
cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab);
|
||||
const bool force_eos = apply_eos(logits_buf.data(), s + 1);
|
||||
cb0 = force_eos ? eng->codec_eos
|
||||
: sampler_sample(talker_sampler, logits_buf.data(), n_vocab);
|
||||
const float* hh = llama_get_embeddings_ith(eng->talker_ctx, -1);
|
||||
if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float));
|
||||
N_done = s + 1;
|
||||
|
|
|
|||
Loading…
Reference in New Issue