From 930f3c8c1fbb1ba87fabee3bd4f482371f6e7696 Mon Sep 17 00:00:00 2001 From: Richard Loyer Date: Thu, 28 May 2026 21:54:21 +0200 Subject: [PATCH] =?UTF-8?q?chantier=20B=20TTS=20#11:=20libkazeia=5Ftts.so?= =?UTF-8?q?=20+=20Kotlin=20wrapper,=20JNI=20valid=C3=A9=20bout-en-bout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor: tts_engine.{h,cpp} = API publique de l'engine TTS (load une fois, synthesize N fois, free). État load-once dans struct TtsEngine (talker model+ctx, CPState, Decoder, KzTextTokenizer, fixtures + embeds spéciaux + role_proj pré-calculé). KV cache talker reset via llama_memory_clear au début de chaque synthesize -> appels indépendants. tts_pipeline.cpp devient un thin CLI dessus (refonte sans changement de sortie : WAV md5 identique au pré-refactor sur 'Bonjour je m'appelle Kazeia'). JNI bridge: kazeia_tts_jni.cpp expose 3 fonctions : Java_com_kazeia_tts_TtsJni_nativeLoad / Synthesize / Free Signatures alignées avec TtsEngine.kt (companion loadLibrary 'kazeia_tts'). Build: build_kazeia_tts.sh -> b-jni/libkazeia_tts.so (~900 KB). Test_engine_2calls (sans JNI) : 3 synth sur même instance, KV reset OK, 2 appels identiques -> WAV md5 identique, appel 3 différent -> codes différents. Test_jni_tts (harness JNI sans VM Java) : dlopen libkazeia_tts.so, JNIEnv mock minimal (GetStringUTFChars/Release + NewIntArray/SetIntArrayRegion), appels load + 2 synth + free. WAV md5 identiques aux runs directs. exit=0. Empreinte tablette mesurée: ~3.5 GB par instance (talker f32 1.6 GB + CP f16 mixte 250 MB + decoder 325 MB + vocab Qwen3 vocab_only 50 MB + fixtures text_embed/tp_* 1.2 GB). Une instance par process. RTF stable autour de 3.0 (CPU 6t), inchangé vs avant refonte. Reste sur la liste initiale : tuning sampling (itératif à l'oreille). Co-Authored-By: Claude Opus 4.7 (1M context) --- dist/build_kazeia_tts.sh | 43 +++ dist/build_test_engine_2calls.sh | 21 ++ dist/build_tts_pipeline.sh | 1 + dist/jni/TtsEngine.kt | 89 ++++++ dist/jni/kazeia_tts_jni.cpp | 105 +++++++ dist/jni/test_engine_2calls.cpp | 49 +++ dist/jni/test_jni_tts.cpp | 120 ++++++++ dist/jni/tts_engine.cpp | 492 +++++++++++++++++++++++++++++++ dist/jni/tts_engine.h | 65 ++++ dist/jni/tts_pipeline.cpp | 485 +++++------------------------- 10 files changed, 1051 insertions(+), 419 deletions(-) create mode 100755 dist/build_kazeia_tts.sh create mode 100755 dist/build_test_engine_2calls.sh create mode 100644 dist/jni/TtsEngine.kt create mode 100644 dist/jni/kazeia_tts_jni.cpp create mode 100644 dist/jni/test_engine_2calls.cpp create mode 100644 dist/jni/test_jni_tts.cpp create mode 100644 dist/jni/tts_engine.cpp create mode 100644 dist/jni/tts_engine.h diff --git a/dist/build_kazeia_tts.sh b/dist/build_kazeia_tts.sh new file mode 100755 index 0000000..4977857 --- /dev/null +++ b/dist/build_kazeia_tts.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Build libkazeia_tts.so (SHARED, JNI) + test_jni_tts (harness standalone). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +NDK_ROOT="${ANDROID_NDK_ROOT:-/opt/Kazeia/android-ndk-r27d}" +CXX="$NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang++" +JNI="$HERE/jni" +INC="$HERE/include" +LIB="$HERE/lib" +DEC="/opt/Kazeia/kazeia-tts-decoder-ggml" +mkdir -p "$HERE/b-jni" + +CFLAGS=( -std=c++17 -O3 -fPIC + -march=armv8.6-a+i8mm+bf16+dotprod+fp16 + -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare + -I"$INC" -I"$DEC/src" +) +SRCS=( + "$JNI/kazeia_tts_jni.cpp" + "$JNI/tts_engine.cpp" + "$JNI/cp_inference.cpp" + "$JNI/sampler.cpp" + "$JNI/kazeia_text_tokenizer.cpp" +) +LIBS=( + "$DEC/build-android-engine/libqwen3tts-decoder.a" + -L"$LIB" -lllama -lggml -lggml-base -lggml-cpu + -Wl,-rpath,'$ORIGIN' + -llog -ldl -lm +) + +echo "== building libkazeia_tts.so ==" +"$CXX" "${CFLAGS[@]}" -shared "${SRCS[@]}" "${LIBS[@]}" -o "$HERE/b-jni/libkazeia_tts.so" +ls -la "$HERE/b-jni/libkazeia_tts.so" + +# Harness JNI sans VM Java : dlopen + JNIEnv mock minimal, vérifie load/synthesize/free. +echo "== building test_jni_tts (harness sans Java VM) ==" +"$CXX" "${CFLAGS[@]}" \ + "$JNI/test_jni_tts.cpp" \ + -Wl,-rpath,'$ORIGIN' \ + -ldl -lm \ + -o "$HERE/b-jni/test_jni_tts" +ls -la "$HERE/b-jni/test_jni_tts" diff --git a/dist/build_test_engine_2calls.sh b/dist/build_test_engine_2calls.sh new file mode 100755 index 0000000..ba083c1 --- /dev/null +++ b/dist/build_test_engine_2calls.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +NDK_ROOT="${ANDROID_NDK_ROOT:-/opt/Kazeia/android-ndk-r27d}" +CXX="$NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang++" +JNI="$HERE/jni" +INC="$HERE/include" +LIB="$HERE/lib" +DEC="/opt/Kazeia/kazeia-tts-decoder-ggml" +mkdir -p "$HERE/b-jni" +"$CXX" -std=c++17 -O3 -fPIE \ + -march=armv8.6-a+i8mm+bf16+dotprod+fp16 \ + -I"$INC" -I"$DEC/src" \ + "$JNI/test_engine_2calls.cpp" "$JNI/tts_engine.cpp" "$JNI/cp_inference.cpp" \ + "$JNI/sampler.cpp" "$JNI/kazeia_text_tokenizer.cpp" \ + "$DEC/build-android-engine/libqwen3tts-decoder.a" \ + -L"$LIB" -lllama -lggml -lggml-base -lggml-cpu \ + -Wl,-rpath,'$ORIGIN' \ + -llog -ldl -lm \ + -o "$HERE/b-jni/test_engine_2calls" +ls -la "$HERE/b-jni/test_engine_2calls" diff --git a/dist/build_tts_pipeline.sh b/dist/build_tts_pipeline.sh index 904d5bd..ea2310d 100755 --- a/dist/build_tts_pipeline.sh +++ b/dist/build_tts_pipeline.sh @@ -32,6 +32,7 @@ CFLAGS=( INCS=( -I"$INC" -I"$DECODER_INC" ) LIBS=( "$JNI/tts_pipeline.cpp" + "$JNI/tts_engine.cpp" "$JNI/cp_inference.cpp" "$JNI/sampler.cpp" "$JNI/kazeia_text_tokenizer.cpp" diff --git a/dist/jni/TtsEngine.kt b/dist/jni/TtsEngine.kt new file mode 100644 index 0000000..ab7e08f --- /dev/null +++ b/dist/jni/TtsEngine.kt @@ -0,0 +1,89 @@ +package com.kazeia.tts +// Engine TTS Qwen3 in-process (libkazeia_tts.so). Load une fois, synthesize N fois. +// Le bridge natif est dans dist/jni/kazeia_tts_jni.cpp ; build via dist/build_kazeia_tts.sh +// ou dist/CMakeLists.txt. +// +// Empreinte RAM typique : ~3.5 GB (talker f32 ~1.6 GB + CP f16 mixte ~250 MB + +// decoder ~325 MB + vocab Qwen3 ~50 MB + fixtures text_embed/tp_* ~1.2 GB). + +class TtsJni { + external fun nativeLoad( + talkerGguf: String, + vocabGguf: String, + dumpDir: String, + useHtp: Boolean, + nThreads: Int, + cpUseCache: Boolean + ): Long + + external fun nativeSynthesize( + handle: Long, + text: String, + outWavPath: String, + maxSteps: Int, + seed: Int, + cpTemp: Float, cpTopK: Int, cpTopP: Float, cpRepPenalty: Float, + talkerTemp: Float, talkerTopK: Int, talkerTopP: Float, talkerRepPenalty: Float + ): IntArray // [err, frames, total_ms, prefill_ms, talker_ms, cp_ms, decoder_ms] + + external fun nativeFree(handle: Long) + + companion object { init { System.loadLibrary("kazeia_tts") } } +} + +data class TtsResult( + val frames: Int, + val audioMs: Int, + val totalMs: Int, + val prefillMs: Int, + val talkerMs: Int, + val cpMs: Int, + val decoderMs: Int +) + +// Configuration sampling. Défauts validés sur fixture FR (cf project_tts_session_28may). +data class TtsSampling( + val cpTemp: Float = 0.9f, val cpTopK: Int = 50, val cpTopP: Float = 1.0f, val cpRepPenalty: Float = 1.05f, + val talkerTemp: Float = 0.9f, val talkerTopK: Int = 50, val talkerTopP: Float = 1.0f, val talkerRepPenalty: Float = 1.05f +) + +class TtsEngine( + talkerGguf: String, + vocabGguf: String, + dumpDir: String, + useHtp: Boolean = false, + nThreads: Int = 6, + cpUseCache: Boolean = true +) { + private val jni = TtsJni() + private val h = jni.nativeLoad(talkerGguf, vocabGguf, dumpDir, useHtp, nThreads, cpUseCache) + init { require(h != 0L) { "TtsEngine: load FAILED (talker=$talkerGguf, vocab=$vocabGguf, dump=$dumpDir)" } } + + @Throws(IllegalStateException::class) + fun synthesize( + text: String, + outWavPath: String, + maxSteps: Int = 256, + seed: Int = 42, + sampling: TtsSampling = TtsSampling() + ): TtsResult { + val r = jni.nativeSynthesize( + h, text, outWavPath, maxSteps, seed, + sampling.cpTemp, sampling.cpTopK, sampling.cpTopP, sampling.cpRepPenalty, + sampling.talkerTemp, sampling.talkerTopK, sampling.talkerTopP, sampling.talkerRepPenalty + ) + if (r[0] != 0) error("TtsEngine.synthesize err=${r[0]} for text=\"$text\"") + val frames = r[1] + return TtsResult( + frames = frames, + audioMs = frames * 1000 / 12, // 12 frames/s = 80 ms par frame + totalMs = r[2], + prefillMs = r[3], + talkerMs = r[4], + cpMs = r[5], + decoderMs = r[6] + ) + } + + fun release() { jni.nativeFree(h) } +} diff --git a/dist/jni/kazeia_tts_jni.cpp b/dist/jni/kazeia_tts_jni.cpp new file mode 100644 index 0000000..417a598 --- /dev/null +++ b/dist/jni/kazeia_tts_jni.cpp @@ -0,0 +1,105 @@ +// JNI bridge pour libkazeia_tts.so. Thin wrapper autour de tts_engine.{h,cpp}. +// Le handle exposé côté Kotlin est un jlong = (jlong)(TtsEngine*). +// +// Signatures (à matcher dans le .kt) : +// nativeLoad(talkerGguf, vocabGguf, dumpDir, useHtp, nThreads, cpUseCache) : Long +// nativeSynthesize(handle, text, outWavPath, maxSteps, seed, +// cpTemp, cpTopK, cpTopP, cpRepPenalty, +// talkerTemp, talkerTopK, talkerTopP, talkerRepPenalty) +// : IntArray { err, frames, total_ms, prefill_ms, talker_ms, cp_ms, decoder_ms } +// nativeFree(handle) +#include "tts_engine.h" +#include +#include + +namespace { + +// Helper : récupère une C-string UTF-8 depuis un jstring (rendre à java avec release). +struct JStr { + JNIEnv * env; + jstring j; + const char * c; + JStr(JNIEnv* e, jstring s) : env(e), j(s), c(s ? e->GetStringUTFChars(s, nullptr) : nullptr) {} + ~JStr() { if (j && c) env->ReleaseStringUTFChars(j, c); } +}; + +} // namespace + +extern "C" { + +JNIEXPORT jlong JNICALL +Java_com_kazeia_tts_TtsJni_nativeLoad(JNIEnv* env, jobject /*thiz*/, + jstring talker_gguf, jstring vocab_gguf, + jstring dump_dir, + jboolean use_htp, jint n_threads, + jboolean cp_use_cache) { + JStr talker(env, talker_gguf); + JStr vocab (env, vocab_gguf); + JStr dump (env, dump_dir); + TtsEngineLoadCfg lc; + lc.talker_gguf = talker.c; + lc.vocab_gguf = vocab.c; + lc.dump_dir = dump.c; + lc.use_htp = (use_htp == JNI_TRUE); + lc.n_threads = n_threads; + lc.cp_use_cache = (cp_use_cache == JNI_TRUE); + auto * eng = tts_engine_load(lc); + return (jlong)(uintptr_t)eng; +} + +JNIEXPORT jintArray JNICALL +Java_com_kazeia_tts_TtsJni_nativeSynthesize(JNIEnv* env, jobject /*thiz*/, + jlong handle, + jstring text, jstring out_wav_path, + jint max_steps, jint seed, + jfloat cp_temp, jint cp_top_k, jfloat cp_top_p, jfloat cp_rep_penalty, + jfloat talker_temp, jint talker_top_k, jfloat talker_top_p, jfloat talker_rep_penalty) { + auto * eng = (TtsEngine*)(uintptr_t)handle; + JStr text_s(env, text); + JStr wav_s (env, out_wav_path); + + TtsSynthesizeCfg sc; + sc.text = text_s.c; + sc.out_wav_path = wav_s.c; + sc.max_steps = max_steps; + sc.seed = (uint32_t)seed; + sc.cp_temp = cp_temp; + sc.cp_top_k = cp_top_k; + sc.cp_top_p = cp_top_p; + sc.cp_rep_penalty = cp_rep_penalty; + sc.talker_temp = talker_temp; + sc.talker_top_k = talker_top_k; + sc.talker_top_p = talker_top_p; + sc.talker_rep_penalty = talker_rep_penalty; + + TtsSynthesizeResult R{ -100, 0, 0, 0, 0, 0, 0, 0 }; + if (eng) R = tts_engine_synthesize(eng, sc); + + // Pack résultats dans IntArray : + // [0] err + // [1] frames + // [2] total_ms (audio_s codé séparément si besoin = frames/12) + // [3] prefill_ms + // [4] talker_loop_ms + // [5] cp_loop_ms + // [6] decoder_ms + jint out[7]; + out[0] = R.err; + out[1] = R.frames; + out[2] = (jint)(R.total_s * 1000.0); + out[3] = (jint)(R.prefill_s * 1000.0); + out[4] = (jint)(R.talker_loop_s * 1000.0); + out[5] = (jint)(R.cp_loop_s * 1000.0); + out[6] = (jint)(R.decoder_s * 1000.0); + jintArray arr = env->NewIntArray(7); + env->SetIntArrayRegion(arr, 0, 7, out); + return arr; +} + +JNIEXPORT void JNICALL +Java_com_kazeia_tts_TtsJni_nativeFree(JNIEnv* /*env*/, jobject /*thiz*/, jlong handle) { + auto * eng = (TtsEngine*)(uintptr_t)handle; + tts_engine_free(eng); +} + +} // extern "C" diff --git a/dist/jni/test_engine_2calls.cpp b/dist/jni/test_engine_2calls.cpp new file mode 100644 index 0000000..77100f1 --- /dev/null +++ b/dist/jni/test_engine_2calls.cpp @@ -0,0 +1,49 @@ +// Vérifie qu'on peut load une fois puis synthesize 2 fois sur la même instance, +// avec KV cache talker resetée entre les deux. Critère : +// - 2e appel sur la même phrase doit produire le même WAV md5 que le 1er +// (= la KV reset a bien tout effacé). +// - 2e appel sur une phrase différente doit produire des codes différents +// (= la nouvelle phrase est bien tokenisée et synthétisée fresh). +#include "tts_engine.h" +#include +#include +#include + +int main(int argc, char** argv) { + if (argc < 5) { + printf("usage: %s \n", argv[0]); + return 1; + } + TtsEngineLoadCfg lc; + lc.talker_gguf = argv[1]; + lc.vocab_gguf = argv[2]; + lc.dump_dir = argv[3]; + lc.use_htp = false; + lc.n_threads = 6; + lc.cp_use_cache = true; + std::string OUT = argv[4]; + + auto * eng = tts_engine_load(lc); + if (!eng) return 2; + + auto run = [&](const char* text, const char* fname) { + TtsSynthesizeCfg sc; + sc.text = text; + std::string path = OUT + "/" + fname; + sc.out_wav_path = path.c_str(); + sc.max_steps = 256; + sc.seed = 42; + auto R = tts_engine_synthesize(eng, sc); + printf("[%s] err=%d frames=%d audio=%.2fs total=%.2fs RTF=%.2f\n", + fname, R.err, R.frames, R.audio_s, R.total_s, + R.audio_s > 0 ? R.total_s / R.audio_s : 0); + return R.err; + }; + + int e1 = run("Bonjour je m'appelle Kazeia", "out_call1.wav"); + int e2 = run("Bonjour je m'appelle Kazeia", "out_call2.wav"); + int e3 = run("Bonsoir, comment tu te sens ce soir ?", "out_call3.wav"); + + tts_engine_free(eng); + return (e1 == 0 && e2 == 0 && e3 == 0) ? 0 : 3; +} diff --git a/dist/jni/test_jni_tts.cpp b/dist/jni/test_jni_tts.cpp new file mode 100644 index 0000000..63683ec --- /dev/null +++ b/dist/jni/test_jni_tts.cpp @@ -0,0 +1,120 @@ +// Harness JNI sans VM Java : on dlopen libkazeia_tts.so, on récupère les Java_* +// symbols, et on les appelle avec un JNIEnv mock minimal (juste les fonctions que +// kazeia_tts_jni.cpp utilise : GetStringUTFChars, ReleaseStringUTFChars, NewIntArray, +// SetIntArrayRegion). jstring est juste un alias opaque pour const char*. +// +// Permet de valider le bridge JNI sans installer l'app — utile pour CI / régression. +#include +#include +#include +#include // _exit +#include +#include +#include + +namespace { + +// Pour notre mock, jstring = const char* tel quel (alias). +const char * mock_GetStringUTFChars(JNIEnv*, jstring s, jboolean*) { return (const char*)s; } +void mock_ReleaseStringUTFChars(JNIEnv*, jstring, const char*) {} + +// jintArray = pointeur vers une struct {int n; jint data[n]} qu'on alloue. +struct IntArr { int n; jint data[1]; }; +jintArray mock_NewIntArray(JNIEnv*, jsize len) { + auto* a = (IntArr*)malloc(sizeof(int) + (size_t)len * sizeof(jint)); + a->n = len; + return (jintArray)a; +} +void mock_SetIntArrayRegion(JNIEnv*, jintArray arr, jsize start, jsize len, const jint* buf) { + auto* a = (IntArr*)arr; + memcpy(a->data + start, buf, (size_t)len * sizeof(jint)); +} + +struct MockNativeInterface { + // Les offsets et le layout doivent matcher jni.h JNINativeInterface. Plutôt que + // de reproduire toute la struct, on alloue une zone large, et on pose les + // function pointers AUX BONS OFFSETS (cf jni.h: GetStringUTFChars = entry 169, + // ReleaseStringUTFChars = 170, NewIntArray = 187, SetIntArrayRegion = 207). + // Pour rester portable, on utilise directement l'API via le struct JNINativeInterface + // déclaré dans jni.h. + JNINativeInterface iface{}; + JNINativeInterface * ptr; + MockNativeInterface() { + // On laisse tout à nullptr, on remplit juste les méthodes utilisées. + iface.GetStringUTFChars = mock_GetStringUTFChars; + iface.ReleaseStringUTFChars = mock_ReleaseStringUTFChars; + iface.NewIntArray = mock_NewIntArray; + iface.SetIntArrayRegion = mock_SetIntArrayRegion; + ptr = &iface; + } +}; + +} // namespace + +int main(int argc, char** argv) { + if (argc < 6) { + fprintf(stderr,"usage: %s \n", argv[0]); + return 1; + } + const char* lib_path = argv[1]; + const char* talker_gguf = argv[2]; + const char* vocab_gguf = argv[3]; + const char* dump_dir = argv[4]; + const char* out_wav = argv[5]; + + void* dl = dlopen(lib_path, RTLD_NOW | RTLD_LOCAL); + if (!dl) { fprintf(stderr, "dlopen FAIL %s : %s\n", lib_path, dlerror()); return 2; } + + using FnLoad = jlong (*)(JNIEnv*, jobject, jstring, jstring, jstring, jboolean, jint, jboolean); + using FnSynth = jintArray (*)(JNIEnv*, jobject, jlong, jstring, jstring, jint, jint, + jfloat, jint, jfloat, jfloat, jfloat, jint, jfloat, jfloat); + using FnFree = void (*)(JNIEnv*, jobject, jlong); + + auto p_load = (FnLoad)dlsym(dl, "Java_com_kazeia_tts_TtsJni_nativeLoad"); + auto p_synth = (FnSynth)dlsym(dl, "Java_com_kazeia_tts_TtsJni_nativeSynthesize"); + auto p_free = (FnFree)dlsym(dl, "Java_com_kazeia_tts_TtsJni_nativeFree"); + if (!p_load || !p_synth || !p_free) { + fprintf(stderr, "dlsym FAIL : load=%p synth=%p free=%p\n", (void*)p_load, (void*)p_synth, (void*)p_free); + return 3; + } + + MockNativeInterface ifc; + JNIEnv env_buf; + env_buf.functions = ifc.ptr; + JNIEnv* env = &env_buf; + jobject self = nullptr; // pas utilisé dans nos handlers + + jlong h = p_load(env, self, + (jstring)talker_gguf, (jstring)vocab_gguf, (jstring)dump_dir, + /*useHtp=*/JNI_FALSE, /*nThreads=*/6, /*cpUseCache=*/JNI_TRUE); + if (h == 0) { fprintf(stderr, "nativeLoad returned 0\n"); dlclose(dl); return 4; } + fprintf(stderr,"nativeLoad OK -> handle=%p\n", (void*)(uintptr_t)h); + + // 2 appels : même phrase puis phrase différente. + auto run = [&](const char* text, const char* out_path) { + auto* arr = p_synth(env, self, h, + (jstring)text, (jstring)out_path, + /*maxSteps=*/256, /*seed=*/42, + /*cpTemp=*/0.9f, /*cpTopK=*/50, /*cpTopP=*/1.0f, /*cpRepPenalty=*/1.05f, + /*talkerTemp=*/0.9f, /*talkerTopK=*/50, /*talkerTopP=*/1.0f, /*talkerRepPenalty=*/1.05f); + IntArr* a = (IntArr*)arr; + fprintf(stderr,"[%s] err=%d frames=%d total=%dms (pf=%d talker=%d cp=%d dec=%d) -> %s\n", + text, a->data[0], a->data[1], a->data[2], a->data[3], a->data[4], a->data[5], a->data[6], out_path); + int err = a->data[0]; + free(a); + return err; + }; + + std::string out1 = out_wav; out1 += ".1.wav"; + std::string out2 = out_wav; out2 += ".2.wav"; + int e1 = run("Bonjour je m'appelle Kazeia", out1.c_str()); + int e2 = run("Bonsoir, comment tu te sens ce soir ?", out2.c_str()); + + p_free(env, self, h); + fprintf(stderr,"OK e1=%d e2=%d\n", e1, e2); + fflush(stderr); fflush(stdout); + // Pas de dlclose ni return : destructors statiques de libllama/ggml peuvent segfaulter + // après unmap (atexit/global dtors). En contexte app Android, la .so reste chargée + // pour toute la vie du process, donc ce path n'existe pas. _exit court-circuite. + _exit((e1 == 0 && e2 == 0) ? 0 : 5); +} diff --git a/dist/jni/tts_engine.cpp b/dist/jni/tts_engine.cpp new file mode 100644 index 0000000..5134e38 --- /dev/null +++ b/dist/jni/tts_engine.cpp @@ -0,0 +1,492 @@ +#include "tts_engine.h" +#include "llama.h" +#include "ggml-backend.h" +#include "cp_inference.h" +#include "sampler.h" +#include "kazeia_text_tokenizer.h" +#include "decoder.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace kazeia::tts; + +namespace { + +double now_s() { + using clk = std::chrono::steady_clock; + return std::chrono::duration(clk::now().time_since_epoch()).count(); +} + +std::vector read_f32(const std::string& p, size_t n_expected) { + std::ifstream f(p, std::ios::binary | std::ios::ate); + if (!f) { fprintf(stderr, "tts_engine: open %s\n", p.c_str()); return {}; } + size_t n = (size_t)f.tellg() / sizeof(float); + if (n_expected && n != n_expected) { + fprintf(stderr, "tts_engine: %s: %zu f32, attendu %zu\n", p.c_str(), n, n_expected); return {}; + } + f.seekg(0); std::vector v(n); f.read((char*)v.data(), n * sizeof(float)); return v; +} + +ggml_backend_dev_t find_htp() { + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto d = ggml_backend_dev_get(i); + if (!strcmp(ggml_backend_dev_name(d), "HTP0")) return d; + } + return nullptr; +} + +inline float silu(float x) { return x / (1.0f + std::exp(-x)); } + +void linear(const float* W, const float* b, int out_dim, int in_dim, const float* x, float* out) { + for (int m = 0; m < out_dim; ++m) { + float s = b ? b[m] : 0.0f; + const float* wr = W + (size_t)m * in_dim; + for (int k = 0; k < in_dim; ++k) s += wr[k] * x[k]; + out[m] = s; + } +} + +void text_projection(const float* in, int N, + const float* fc1_w, const float* fc1_b, + const float* fc2_w, const float* fc2_b, + float* mid_buf, float* out, + int text_hidden, int hidden) { + for (int n = 0; n < N; ++n) { + linear(fc1_w, fc1_b, text_hidden, text_hidden, in + n * text_hidden, mid_buf); + for (int d = 0; d < text_hidden; ++d) mid_buf[d] = silu(mid_buf[d]); + linear(fc2_w, fc2_b, hidden, text_hidden, mid_buf, out + n * hidden); + } +} + +bool write_wav_pcm16_mono(const char* path, const float* samples, size_t N, int sr = 24000) { + FILE* f = fopen(path, "wb"); + if (!f) return false; + const uint32_t data_bytes = (uint32_t)(N * 2); + const uint32_t riff_size = 36 + data_bytes; + auto w16 = [&](uint16_t v){ fwrite(&v, 2, 1, f); }; + auto w32 = [&](uint32_t v){ fwrite(&v, 4, 1, f); }; + fwrite("RIFF", 1, 4, f); w32(riff_size); fwrite("WAVE", 1, 4, f); + fwrite("fmt ", 1, 4, f); w32(16); w16(1); w16(1); + w32((uint32_t)sr); w32((uint32_t)sr * 2); w16(2); w16(16); + fwrite("data", 1, 4, f); w32(data_bytes); + for (size_t i = 0; i < N; ++i) { + float v = samples[i]; if (v > 1.f) v = 1.f; else if (v < -1.f) v = -1.f; + int16_t pcm = (int16_t)std::lround(v * 32767.0f); + fwrite(&pcm, 2, 1, f); + } + fclose(f); return true; +} + +} // namespace + +struct TtsEngine { + // --- Constantes manifest --- + int text_vocab = 151936, text_hidden = 2048, hidden = 1024; + int tts_bos = 151672, tts_eos = 151673, tts_pad = 151671; + int codec_bos = 2149, codec_eos = 2150, codec_pad = 2148; + int codec_think = 2154, codec_think_bos = 2156, codec_think_eos = 2157; + int lang_fr = 2061; + + // --- Fixtures (load-once) --- + std::vector text_embed; + std::vector tp_fc1_w, tp_fc1_b, tp_fc2_w, tp_fc2_b; + std::vector tok_embd; + std::vector xvector; + + // Embeds spéciaux pré-projetés (constants une fois le modèle chargé) + std::vector spec_proj; // [3 * hidden] = {tts_bos_emb, tts_eos_emb, tts_pad_emb} + std::vector codec_input_emb; // [7 * hidden] = {think, think_bos, lang_fr, think_eos, xvec, codec_pad, codec_bos} + std::vector role_proj; // [3 * hidden] = projection de <|im_start|>, assistant, \n (constant car template fixe) + std::vector codec_pad_emb; // [hidden] = tok_embd[codec_pad] + + // --- Talker --- + llama_model * talker_m = nullptr; + llama_context * talker_ctx = nullptr; + int n_embd = 0; + int n_vocab = 0; + int npe = 1; // 1 (RoPE) ou 4 (M-RoPE/I-MRoPE) + int n_threads = 6; + + // --- CP / Decoder / Tokenizer --- + CPState cp_state; + Decoder decoder; + KzTextTokenizer kz_tok; + bool has_kz_tok = false; + bool cp_use_cache = true; +}; + +// =========================================================================== +// LOAD +// =========================================================================== + +TtsEngine * tts_engine_load(const TtsEngineLoadCfg & cfg) { + if (!cfg.talker_gguf || !cfg.dump_dir) { + fprintf(stderr, "tts_engine_load: talker_gguf et dump_dir requis\n"); return nullptr; + } + + auto eng = new TtsEngine(); + eng->n_threads = cfg.n_threads; + eng->cp_use_cache = cfg.cp_use_cache; + + std::string D = cfg.dump_dir; + if (D.back() != '/') D += '/'; + + // --- 0) Constantes manifest --- + { + std::ifstream f(D + "manifest_text.txt"); + if (!f) { fprintf(stderr, "tts_engine_load: no manifest_text\n"); delete eng; return nullptr; } + std::string line; + auto eq = [&](const char* k){ return line.rfind(k, 0) == 0; }; + auto val = [&](size_t off){ return atoi(line.c_str() + off); }; + while (std::getline(f, line)) { + if (eq("text_vocab_size:")) eng->text_vocab = val(16); + else if (eq("text_hidden_size:")) eng->text_hidden = val(17); + else if (eq("hidden_size:")) eng->hidden = val(12); + else if (eq("tts_bos_token_id:")) eng->tts_bos = val(17); + else if (eq("tts_eos_token_id:")) eng->tts_eos = val(17); + else if (eq("tts_pad_token_id:")) eng->tts_pad = val(17); + else if (eq("codec_bos_id:")) eng->codec_bos = val(13); + else if (eq("codec_eos_id:")) eng->codec_eos = val(13); + else if (eq("codec_pad_id:")) eng->codec_pad = val(13); + else if (eq("codec_think_id:")) eng->codec_think = val(15); + else if (eq("codec_think_bos_id:")) eng->codec_think_bos = val(19); + else if (eq("codec_think_eos_id:")) eng->codec_think_eos = val(19); + else if (eq("codec_language_french:")) eng->lang_fr = val(22); + } + } + + // --- 1) Fixtures --- + const int text_hidden = eng->text_hidden; + const int hidden = eng->hidden; + const int text_vocab = eng->text_vocab; + const double t_load0 = now_s(); + eng->text_embed = read_f32(D + "text_embed.bin", (size_t)text_vocab * text_hidden); + eng->tp_fc1_w = read_f32(D + "tp_fc1_w.bin", (size_t)text_hidden * text_hidden); + eng->tp_fc1_b = read_f32(D + "tp_fc1_b.bin", (size_t)text_hidden); + eng->tp_fc2_w = read_f32(D + "tp_fc2_w.bin", (size_t)hidden * text_hidden); + eng->tp_fc2_b = read_f32(D + "tp_fc2_b.bin", (size_t)hidden); + eng->tok_embd = read_f32(D + "talker_tok_embd.bin", (size_t)3072 * hidden); + eng->xvector = read_f32(D + "damien_xvector.bin", (size_t)hidden); + if (eng->text_embed.empty() || eng->tp_fc1_w.empty() || eng->tok_embd.empty() || eng->xvector.empty()) { + fprintf(stderr, "tts_engine_load: fixtures FAIL\n"); delete eng; return nullptr; + } + + // --- 2) Backend talker (avant le 1er init backend pour HMX off, etc.) --- + setenv("GGML_HEXAGON_USE_HMX", "0", 1); + llama_backend_init(); + + // --- 3) Tokenizer optionnel (vocab_only -> ~50 MB RAM) --- + if (cfg.vocab_gguf && *cfg.vocab_gguf) { + if (!kz_tok_load(eng->kz_tok, cfg.vocab_gguf)) { + fprintf(stderr, "tts_engine_load: vocab load FAIL %s\n", cfg.vocab_gguf); + delete eng; return nullptr; + } + eng->has_kz_tok = true; + } + + // --- 4) Talker (libllama) --- + auto mp = llama_model_default_params(); + ggml_backend_dev_t devs[2] = { cfg.use_htp ? find_htp() : nullptr, nullptr }; + if (devs[0]) { mp.n_gpu_layers = 99; mp.devices = devs; fprintf(stderr, "talker: HTP0\n"); } + else { mp.n_gpu_layers = 0; fprintf(stderr, "talker: CPU\n"); } + eng->talker_m = llama_model_load_from_file(cfg.talker_gguf, mp); + if (!eng->talker_m) { fprintf(stderr, "talker load FAIL %s\n", cfg.talker_gguf); delete eng; return nullptr; } + + auto cp = llama_context_default_params(); + // Borne ctx généreuse : prefill max ~50 + max_steps 512 + marge. Pas critique en CPU. + cp.n_ctx = 1024; + cp.n_batch = 1024; + cp.n_threads = cfg.n_threads; + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + cp.embeddings = true; + eng->talker_ctx = llama_init_from_model(eng->talker_m, cp); + if (!eng->talker_ctx) { fprintf(stderr, "talker ctx FAIL\n"); delete eng; return nullptr; } + eng->n_embd = llama_model_n_embd(eng->talker_m); + eng->n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(eng->talker_m)); + if (eng->n_embd != hidden || eng->n_vocab != 3072) { + fprintf(stderr, "talker dims mismatch (n_embd=%d, n_vocab=%d, attendu %d / 3072)\n", + eng->n_embd, eng->n_vocab, hidden); delete eng; return nullptr; + } + auto rt = llama_model_rope_type(eng->talker_m); + eng->npe = (rt == LLAMA_ROPE_TYPE_MROPE || rt == LLAMA_ROPE_TYPE_IMROPE) ? 4 : 1; + + // --- 5) CP --- + eng->cp_state.n_threads = cfg.n_threads; + if (!cp_load(eng->cp_state, + (D + "cp_f16.gguf").c_str(), + (D + "cp_heads.bin").c_str(), + (D + "cp_codec_embs.bin").c_str())) { + fprintf(stderr, "CP load FAIL\n"); delete eng; return nullptr; + } + + // --- 6) Decoder --- + if (!eng->decoder.load((D + "qwen3tts_decoder.gguf").c_str())) { + fprintf(stderr, "decoder load FAIL\n"); delete eng; return nullptr; + } + + // --- 7) Pre-compute des embeds constants --- + std::vector mid(text_hidden); + + // special tokens projetés (tts_bos/eos/pad) + std::vector spec_text(3 * text_hidden); + int sp[3] = { eng->tts_bos, eng->tts_eos, eng->tts_pad }; + for (int i = 0; i < 3; ++i) + std::memcpy(spec_text.data() + i * text_hidden, + eng->text_embed.data() + (size_t)sp[i] * text_hidden, + text_hidden * sizeof(float)); + eng->spec_proj.resize(3 * hidden); + text_projection(spec_text.data(), 3, + eng->tp_fc1_w.data(), eng->tp_fc1_b.data(), + eng->tp_fc2_w.data(), eng->tp_fc2_b.data(), + mid.data(), eng->spec_proj.data(), text_hidden, hidden); + + // codec prefix [think, think_bos, lang_fr, think_eos, x_vec, codec_pad, codec_bos] + int codec_prefill[4] = { eng->codec_think, eng->codec_think_bos, eng->lang_fr, eng->codec_think_eos }; + eng->codec_input_emb.resize(7 * hidden); + for (int i = 0; i < 4; ++i) + std::memcpy(eng->codec_input_emb.data() + i * hidden, + eng->tok_embd.data() + (size_t)codec_prefill[i] * hidden, + hidden * sizeof(float)); + std::memcpy(eng->codec_input_emb.data() + 4 * hidden, eng->xvector.data(), hidden * sizeof(float)); + std::memcpy(eng->codec_input_emb.data() + 5 * hidden, + eng->tok_embd.data() + (size_t)eng->codec_pad * hidden, hidden * sizeof(float)); + std::memcpy(eng->codec_input_emb.data() + 6 * hidden, + eng->tok_embd.data() + (size_t)eng->codec_bos * hidden, hidden * sizeof(float)); + + // role tokens (<|im_start|>, assistant, \n) — projection constante car template fixe. + // Pré-calculé une fois, indépendant du texte d'entrée. + const int role_ids[3] = { 151644, 77091, 198 }; + std::vector role_text(3 * text_hidden); + for (int i = 0; i < 3; ++i) + std::memcpy(role_text.data() + i * text_hidden, + eng->text_embed.data() + (size_t)role_ids[i] * text_hidden, + text_hidden * sizeof(float)); + eng->role_proj.resize(3 * hidden); + text_projection(role_text.data(), 3, + eng->tp_fc1_w.data(), eng->tp_fc1_b.data(), + eng->tp_fc2_w.data(), eng->tp_fc2_b.data(), + mid.data(), eng->role_proj.data(), text_hidden, hidden); + + // codec_pad_emb (utilisé partout dans le body) + eng->codec_pad_emb.assign(eng->tok_embd.data() + (size_t)eng->codec_pad * hidden, + eng->tok_embd.data() + (size_t)(eng->codec_pad + 1) * hidden); + + fprintf(stderr, "tts_engine_load: %.2fs (talker_n_embd=%d, npe=%d, kz_tok=%d, cp_cache=%d)\n", + now_s() - t_load0, eng->n_embd, eng->npe, (int)eng->has_kz_tok, (int)cfg.cp_use_cache); + return eng; +} + +// =========================================================================== +// SYNTHESIZE +// =========================================================================== + +TtsSynthesizeResult tts_engine_synthesize(TtsEngine * eng, const TtsSynthesizeCfg & cfg) { + TtsSynthesizeResult R{}; + if (!eng || !cfg.text || !cfg.out_wav_path) { R.err = -1; return R; } + if (!eng->has_kz_tok) { + fprintf(stderr, "tts_engine_synthesize: pas de tokenizer chargé (vocab_gguf manquant au load)\n"); + R.err = -2; return R; + } + const int text_hidden = eng->text_hidden; + const int hidden = eng->hidden; + const int n_embd = eng->n_embd; + const int n_vocab = eng->n_vocab; + + // --- 1) Tokenize --- + auto input_ids = kz_tok_encode_tts_prompt(eng->kz_tok, cfg.text); + const int input_ids_len = (int)input_ids.size(); + if (input_ids_len < 8) { + fprintf(stderr, "tts_engine_synthesize: tokens trop courts (%d, min=8)\n", input_ids_len); + R.err = -3; return R; + } + const int Nt = input_ids_len - 5 - 3; // role(3) + body(Nt) + trailing(5) + + // --- 2) Body projection --- + std::vector mid(text_hidden); + std::vector body_text((size_t)Nt * text_hidden); + for (int i = 0; i < Nt; ++i) + std::memcpy(body_text.data() + i * text_hidden, + eng->text_embed.data() + (size_t)input_ids[3 + i] * text_hidden, + text_hidden * sizeof(float)); + std::vector body_proj((size_t)Nt * hidden); + text_projection(body_text.data(), Nt, + eng->tp_fc1_w.data(), eng->tp_fc1_b.data(), + eng->tp_fc2_w.data(), eng->tp_fc2_b.data(), + mid.data(), body_proj.data(), text_hidden, hidden); + + // --- 3) Assemble prefill --- + const float* tts_bos_emb = eng->spec_proj.data() + 0 * hidden; + const float* tts_eos_emb = eng->spec_proj.data() + 1 * hidden; + const float* tts_pad_emb = eng->spec_proj.data() + 2 * hidden; + const float* codec_pad_emb = eng->codec_pad_emb.data(); + const int T_prefill = 3 + 6 + (Nt + 1) + 1; + + std::vector prefill((size_t)T_prefill * hidden, 0.0f); + std::memcpy(prefill.data(), eng->role_proj.data(), 3 * hidden * sizeof(float)); + for (int i = 0; i < 5; ++i) { + const float* ce = eng->codec_input_emb.data() + i * hidden; + float* p = prefill.data() + (3 + i) * hidden; + for (int d = 0; d < hidden; ++d) p[d] = tts_pad_emb[d] + ce[d]; + } + { const float* ce = eng->codec_input_emb.data() + 5 * hidden; + float* p = prefill.data() + 8 * hidden; + for (int d = 0; d < hidden; ++d) p[d] = tts_bos_emb[d] + ce[d]; } + for (int i = 0; i < Nt; ++i) { + const float* bp = body_proj.data() + i * hidden; + float* p = prefill.data() + (9 + i) * hidden; + for (int d = 0; d < hidden; ++d) p[d] = bp[d] + codec_pad_emb[d]; + } + { float* p = prefill.data() + (9 + Nt) * hidden; + for (int d = 0; d < hidden; ++d) p[d] = tts_eos_emb[d] + codec_pad_emb[d]; } + { const float* cbe = eng->tok_embd.data() + (size_t)eng->codec_bos * hidden; + float* p = prefill.data() + (10 + Nt) * hidden; + for (int d = 0; d < hidden; ++d) p[d] = tts_pad_emb[d] + cbe[d]; } + + // --- 4) Reset KV talker (chaque synthèse repart de pos=0) --- + llama_memory_clear(llama_get_memory(eng->talker_ctx), true); + + // --- 5) Configure les samplers --- + eng->cp_state.sampler.temp = cfg.cp_temp; + eng->cp_state.sampler.top_k = cfg.cp_top_k; + eng->cp_state.sampler.top_p = cfg.cp_top_p; + eng->cp_state.sampler.rep_penalty = cfg.cp_rep_penalty; + eng->cp_state.sampler.rep_window = 16; + sampler_seed(eng->cp_state.sampler, cfg.seed + 1); + + Sampler talker_sampler{}; + talker_sampler.temp = cfg.talker_temp; + talker_sampler.top_k = cfg.talker_top_k; + talker_sampler.top_p = cfg.talker_top_p; + talker_sampler.rep_penalty = cfg.talker_rep_penalty; + talker_sampler.rep_window = 64; + sampler_seed(talker_sampler, cfg.seed); + + // --- 6) Talker prefill --- + const double t_pfill0 = now_s(); + { + const int npe = eng->npe; + std::vector pos(T_prefill * npe, 0); + std::vector nsd(T_prefill, 1); + std::vector sid0(T_prefill, 0); + std::vector sids(T_prefill); + std::vector lg(T_prefill, 0); + for (int i = 0; i < T_prefill; ++i) { + if (npe == 4) { pos[i] = i; pos[T_prefill + i] = i; pos[2*T_prefill + i] = i; pos[3*T_prefill + i] = 0; } + else { pos[i] = i; } + sids[i] = &sid0[i]; + } + lg[T_prefill - 1] = 1; + llama_batch b{}; + b.n_tokens = T_prefill; b.embd = prefill.data(); + b.pos = pos.data(); b.n_seq_id = nsd.data(); b.seq_id = sids.data(); b.logits = lg.data(); + if (llama_decode(eng->talker_ctx, b) != 0) { + fprintf(stderr, "tts_engine_synthesize: prefill FAIL\n"); R.err = -4; return R; + } + } + R.prefill_s = now_s() - t_pfill0; + + // --- 7) Decode loop (CB0 -> CP -> next_embed -> talker step) --- + std::vector logits_buf(n_vocab); + { + const float* lp = llama_get_logits_ith(eng->talker_ctx, -1); + memcpy(logits_buf.data(), lp, n_vocab * sizeof(float)); + } + int cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab); + std::vector hidden_for_cp(n_embd); + { const float* hh = llama_get_embeddings_ith(eng->talker_ctx, -1); + if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float)); } + + std::vector codes_engine; + codes_engine.reserve((size_t)cfg.max_steps * 16); + int N_done = 0; + const double t_loop0 = now_s(); + double t_decode_total = 0, t_cp_total = 0; + + for (int s = 0; s < cfg.max_steps; ++s) { + if (cb0 == eng->codec_eos) break; + codes_engine.push_back(cb0); + + const double tcp0 = now_s(); + const float* cb0_emb = eng->tok_embd.data() + (size_t)cb0 * n_embd; + int32_t cb15[15]; + if (eng->cp_use_cache) cp_predict_cached(eng->cp_state, hidden_for_cp.data(), cb0_emb, cb15); + else cp_predict (eng->cp_state, hidden_for_cp.data(), cb0_emb, cb15); + t_cp_total += now_s() - tcp0; + for (int i = 0; i < 15; ++i) codes_engine.push_back(cb15[i]); + + std::vector next_embed(n_embd, 0.0f); + const float* e_cb0 = eng->tok_embd.data() + (size_t)cb0 * n_embd; + for (int d = 0; d < n_embd; ++d) next_embed[d] = e_cb0[d]; + for (int i = 1; i < 16; ++i) { + int code = cb15[i - 1]; + const float* e = eng->cp_state.codec_embs.data() + ((size_t)(i-1) * 2048 + code) * n_embd; + for (int d = 0; d < n_embd; ++d) next_embed[d] += e[d]; + } + for (int d = 0; d < n_embd; ++d) next_embed[d] += tts_pad_emb[d]; + + const int npe = eng->npe; + llama_pos pos1[4] = {0,0,0,0}; + const llama_pos p = T_prefill + s; + if (npe == 4) { pos1[0] = p; pos1[1] = p; pos1[2] = p; pos1[3] = 0; } + else { pos1[0] = p; } + int32_t nn = 1; llama_seq_id sd = 0; llama_seq_id* sp = &sd; int8_t l = 1; + llama_batch b{}; + b.n_tokens = 1; b.embd = next_embed.data(); + b.pos = pos1; b.n_seq_id = &nn; b.seq_id = &sp; b.logits = &l; + const double td0 = now_s(); + if (llama_decode(eng->talker_ctx, b) != 0) { R.err = -5; break; } + t_decode_total += now_s() - td0; + + { + const float* lp = llama_get_logits_ith(eng->talker_ctx, -1); + memcpy(logits_buf.data(), lp, n_vocab * sizeof(float)); + } + cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab); + const float* hh = llama_get_embeddings_ith(eng->talker_ctx, -1); + if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float)); + N_done = s + 1; + } + R.talker_loop_s = (now_s() - t_loop0) - t_cp_total; // ne pas double-compter CP + R.cp_loop_s = t_cp_total; + (void)t_decode_total; // déjà compris dans talker_loop_s + R.frames = N_done; + R.audio_s = N_done / 12.0; + + // --- 8) Decoder -> WAV --- + std::vector codes_dec(16 * N_done); + for (int t = 0; t < N_done; ++t) + for (int c = 0; c < 16; ++c) + codes_dec[c * N_done + t] = codes_engine[t * 16 + c]; + + const double t_dec0 = now_s(); + auto wav = eng->decoder.forward(codes_dec, N_done); + R.decoder_s = now_s() - t_dec0; + + if (!write_wav_pcm16_mono(cfg.out_wav_path, wav.data(), wav.size(), 24000)) { + fprintf(stderr, "tts_engine_synthesize: WAV write FAIL %s\n", cfg.out_wav_path); + R.err = -6; return R; + } + + R.total_s = R.prefill_s + R.talker_loop_s + R.cp_loop_s + R.decoder_s; + return R; +} + +// =========================================================================== +// FREE +// =========================================================================== + +void tts_engine_free(TtsEngine * eng) { + if (!eng) return; + cp_free(eng->cp_state); + if (eng->talker_ctx) llama_free(eng->talker_ctx); + if (eng->talker_m) llama_model_free(eng->talker_m); + if (eng->has_kz_tok) kz_tok_free(eng->kz_tok); + delete eng; +} diff --git a/dist/jni/tts_engine.h b/dist/jni/tts_engine.h new file mode 100644 index 0000000..ef74672 --- /dev/null +++ b/dist/jni/tts_engine.h @@ -0,0 +1,65 @@ +// Engine TTS Qwen3 in-process : load une fois (modèles + fixtures + tokenizer), +// synthesize N fois (texte FR -> WAV). Conçu pour être appelé depuis : +// - tts_pipeline.cpp (CLI dev) +// - kazeia_tts_jni.cpp (bridge Kotlin) +// - test_jni_tts.cpp (harness sans VM Java) +// +// État load-once (poids talker ~1.6 GB, CP ~250 MB, decoder ~325 MB, vocab ~50 MB, +// fixtures text_embed ~1.2 GB) = ~3.5 GB cumulé sur tablette. Une seule instance +// par process recommandée. +#pragma once +#include +#include + +struct TtsEngine; // opaque + +struct TtsEngineLoadCfg { + const char * talker_gguf = nullptr; // talker_f32.gguf + const char * vocab_gguf = nullptr; // ex: Qwen3-4B-Q4_0.gguf (vocab_only) ; nullptr -> pas de tokenize + const char * dump_dir = nullptr; // contient cp_f16.gguf, cp_heads.bin, cp_codec_embs.bin, + // qwen3tts_decoder.gguf, text_embed.bin, tp_fc1/2_*, talker_tok_embd.bin, + // damien_xvector.bin, manifest_text.txt + bool use_htp = false; // true -> HTP0 prefill (option C), false -> CPU pur + int n_threads = 6; + bool cp_use_cache = true; // KV cache CP (cp_predict_cached) ; false -> cp_predict oracle +}; + +struct TtsSynthesizeCfg { + const char * text = nullptr; + const char * out_wav_path = nullptr; + int max_steps = 256; + uint32_t seed = 42; + + // Sampling (par défaut = défauts in-house validés sur fixture FR) + float cp_temp = 0.9f; + int cp_top_k = 50; + float cp_top_p = 1.0f; + float cp_rep_penalty = 1.05f; + + float talker_temp = 0.9f; + int talker_top_k = 50; + float talker_top_p = 1.0f; + float talker_rep_penalty = 1.05f; +}; + +struct TtsSynthesizeResult { + int err = 0; // 0=ok ; <0 erreur (load/tokenize/decode) + int frames = 0; // N frames audio (12.5 Hz) + double audio_s = 0; // = frames/12 + double total_s = 0; // prefill + talker_loop + decoder (hors writing WAV) + double prefill_s = 0; + double talker_loop_s = 0; + double cp_loop_s = 0; + double decoder_s = 0; +}; + +// Charge tout. Retourne nullptr en cas d'échec. +// IMPORTANT : llama_backend_init() est appelé en interne (idempotent). +TtsEngine * tts_engine_load(const TtsEngineLoadCfg & cfg); + +// Synthétise une phrase. Réinitialise la KV cache talker avant le prefill, donc +// les appels sont indépendants (pas d'état conversationnel). +TtsSynthesizeResult tts_engine_synthesize(TtsEngine * eng, const TtsSynthesizeCfg & cfg); + +// Libère tout. +void tts_engine_free(TtsEngine * eng); diff --git a/dist/jni/tts_pipeline.cpp b/dist/jni/tts_pipeline.cpp index 5fa2e66..3f4bfee 100644 --- a/dist/jni/tts_pipeline.cpp +++ b/dist/jni/tts_pipeline.cpp @@ -1,443 +1,90 @@ -// Pipeline TTS bout-en-bout EN UN SEUL BINAIRE sur tablette : -// texte (input_ids déjà tokenisés) + x_vector -// -> construction prefill_embeds (text_projection + tok_embd + spéciaux) -// -> talker engine (libllama, M-RoPE, embeds-mode, Patch 1+2) -// -> sampling greedy CB0 + CP (cp_inference, recompute bit-exact) -// -> codes [T, 16] -// -> decoder ggml (libqwen3tts-decoder rebuilt vs ql/ggml) -// -> PCM 24 kHz WAV +// CLI dev autour de tts_engine. La logique est maintenant dans tts_engine.{h,cpp} +// (réutilisée par le JNI). Ce binaire reste pour les bench A/B et la régression. // -// Aucune dépendance Python à l'exécution. Tokenizer text + speaker encoder restent -// offline (input_ids pré-calculé pour la phrase, x_vector pré-calculé pour la voix). +// Usage: +// tts_pipeline [cpu|htp] [max_steps] // -// Usage: tts_pipeline [cpu|htp] [max_steps] +// Variables d'environnement : +// KZTTS_TEXT, KZTTS_VOCAB_GGUF : texte arbitraire au lieu de input_ids_full.bin (fixture) +// (le mode fixture est conservé pour bench/régression) +// KZTTS_CP_CACHE : 1 (défaut) = cp_predict_cached, 0 = cp_predict oracle +// KZTTS_SEED : seed sampling (défaut 42) +// KZTTS_THREADS : threads CPU (défaut 6) +// KZTTS_TEMP / TOPK / TOPP / REPP : sampling Talker +// KZTTS_CP_TEMP / CP_TOPK / CP_TOPP / CP_REPP : sampling CP +#include "tts_engine.h" +#include "kazeia_text_tokenizer.h" +#include "llama.h" + #include #include #include -#include -#include -#include -#include #include -#include -#include "llama.h" -#include "ggml-backend.h" -#include "cp_inference.h" -#include "sampler.h" -#include "kazeia_text_tokenizer.h" -#include "decoder.h" // Kazeia decoder ggml (linké via libqwen3tts-decoder.a) +#include +#include -using namespace kazeia::tts; - -static double now_s() { - using clk = std::chrono::steady_clock; - return std::chrono::duration(clk::now().time_since_epoch()).count(); -} - -static std::vector read_f32(const std::string& p, size_t n_expected) { - std::ifstream f(p, std::ios::binary | std::ios::ate); - if (!f) { fprintf(stderr, "open %s\n", p.c_str()); exit(1); } - size_t n = (size_t)f.tellg() / sizeof(float); - if (n_expected && n != n_expected) { fprintf(stderr, "%s: %zu f32, attendu %zu\n", p.c_str(), n, n_expected); exit(1); } - f.seekg(0); std::vector v(n); f.read((char*)v.data(), n * sizeof(float)); return v; -} -static std::vector read_i32(const std::string& p, size_t n_expected) { - std::ifstream f(p, std::ios::binary | std::ios::ate); - if (!f) { fprintf(stderr, "open %s\n", p.c_str()); exit(1); } - size_t n = (size_t)f.tellg() / sizeof(int32_t); - if (n_expected && n != n_expected) { fprintf(stderr, "%s: %zu i32, attendu %zu\n", p.c_str(), n, n_expected); exit(1); } - f.seekg(0); std::vector v(n); f.read((char*)v.data(), n * sizeof(int32_t)); return v; -} -static ggml_backend_dev_t find_htp() { - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - auto d = ggml_backend_dev_get(i); - if (!strcmp(ggml_backend_dev_name(d), "HTP0")) return d; - } - return nullptr; -} -static inline float silu(float x) { return x / (1.0f + std::exp(-x)); } -static void linear(const float* W, const float* b, int out_dim, int in_dim, const float* x, float* out) { - for (int m = 0; m < out_dim; ++m) { - float s = b ? b[m] : 0.0f; - const float* wr = W + (size_t)m * in_dim; - for (int k = 0; k < in_dim; ++k) s += wr[k] * x[k]; - out[m] = s; - } -} -static void text_projection(const float* in, int N, - const float* fc1_w, const float* fc1_b, - const float* fc2_w, const float* fc2_b, - float* mid_buf, float* out) { - for (int n = 0; n < N; ++n) { - linear(fc1_w, fc1_b, 2048, 2048, in + n * 2048, mid_buf); - for (int d = 0; d < 2048; ++d) mid_buf[d] = silu(mid_buf[d]); - linear(fc2_w, fc2_b, 1024, 2048, mid_buf, out + n * 1024); - } -} - -static bool write_wav_pcm16_mono(const char* path, const float* samples, size_t N, int sr = 24000) { - FILE* f = fopen(path, "wb"); - if (!f) return false; - const uint32_t data_bytes = (uint32_t)(N * 2); - const uint32_t riff_size = 36 + data_bytes; - auto w16 = [&](uint16_t v){ fwrite(&v, 2, 1, f); }; - auto w32 = [&](uint32_t v){ fwrite(&v, 4, 1, f); }; - fwrite("RIFF", 1, 4, f); w32(riff_size); fwrite("WAVE", 1, 4, f); - fwrite("fmt ", 1, 4, f); w32(16); w16(1); w16(1); // pcm mono - w32((uint32_t)sr); w32((uint32_t)sr * 2); w16(2); w16(16); - fwrite("data", 1, 4, f); w32(data_bytes); - for (size_t i = 0; i < N; ++i) { - float v = samples[i]; if (v > 1.f) v = 1.f; else if (v < -1.f) v = -1.f; - int16_t pcm = (int16_t)std::lround(v * 32767.0f); - fwrite(&pcm, 2, 1, f); - } - fclose(f); return true; -} +static float env_f(const char* k, float dflt) { const char* v = getenv(k); return v ? (float)atof(v) : dflt; } +static int env_i(const char* k, int dflt) { const char* v = getenv(k); return v ? atoi(v) : dflt; } int main(int argc, char** argv) { if (argc < 4) { printf("usage: %s [cpu|htp] [max_steps]\n", argv[0]); - printf(" Texte arbitraire (au lieu de input_ids_full.bin dumpé) :\n"); - printf(" KZTTS_VOCAB_GGUF=/path/to/qwen3.gguf KZTTS_TEXT=\"phrase libre\" %s ...\n", argv[0]); + printf(" Texte arbitraire :\n"); + printf(" KZTTS_VOCAB_GGUF=/path/qwen3.gguf KZTTS_TEXT=\"phrase libre\" %s ...\n", argv[0]); + printf(" Mode fixture (input_ids_full.bin) : aucune variable d'env requise.\n"); return 1; } - const char* gguf = argv[1]; + const char* talker_gguf = argv[1]; std::string D = argv[2]; if (D.back() != '/') D += '/'; - const char* OUT_WAV = argv[3]; - bool force_cpu = (argc >= 5 && !strcmp(argv[4], "cpu")); - int max_steps_arg = (argc >= 6) ? atoi(argv[5]) : 64; - // Texte arbitraire : si KZTTS_TEXT et KZTTS_VOCAB_GGUF sont posés, on tokenize la phrase - // au lieu de relire input_ids_full.bin. Vérifié bit-exact contre le golden HF tokenizer - // sur "Bonjour je m'appelle Kazeia" via test_tokenizer. - const char * kz_text = getenv("KZTTS_TEXT"); - const char * kz_vocab_gguf = getenv("KZTTS_VOCAB_GGUF"); - const bool use_kz_tok = (kz_text && kz_vocab_gguf && *kz_text && *kz_vocab_gguf); + const char* out_wav = argv[3]; + const bool use_htp = (argc >= 5 && !strcmp(argv[4], "htp")); + const int max_steps = (argc >= 6) ? atoi(argv[5]) : 256; + const char* kz_text = getenv("KZTTS_TEXT"); + const char* kz_vocab_gguf = getenv("KZTTS_VOCAB_GGUF"); - // ----- 0) Constants from manifest_text - int text_vocab = 151936, text_hidden = 2048, hidden = 1024; - int tts_bos = 151672, tts_eos = 151673, tts_pad = 151671; - int codec_bos = 2149, codec_eos = 2150, codec_pad = 2148; - int codec_think = 2154, codec_nothink = 2155, codec_think_bos = 2156, codec_think_eos = 2157; - int lang_fr = 2061; - int input_ids_len = 16; - { - std::ifstream f(D + "manifest_text.txt"); if (!f) { fprintf(stderr, "no manifest_text\n"); return 1; } - std::string line; - auto eq = [&](const char* k){ return line.rfind(k, 0) == 0; }; - auto val = [&](size_t off){ return atoi(line.c_str() + off); }; - while (std::getline(f, line)) { - if (eq("text_vocab_size:")) text_vocab = val(16); - else if (eq("text_hidden_size:")) text_hidden = val(17); - else if (eq("hidden_size:")) hidden = val(12); - else if (eq("tts_bos_token_id:")) tts_bos = val(17); - else if (eq("tts_eos_token_id:")) tts_eos = val(17); - else if (eq("tts_pad_token_id:")) tts_pad = val(17); - else if (eq("codec_bos_id:")) codec_bos = val(13); - else if (eq("codec_eos_id:")) codec_eos = val(13); - else if (eq("codec_pad_id:")) codec_pad = val(13); - else if (eq("codec_think_id:")) codec_think = val(15); - else if (eq("codec_nothink_id:")) codec_nothink = val(17); - else if (eq("codec_think_bos_id:")) codec_think_bos = val(19); - else if (eq("codec_think_eos_id:")) codec_think_eos = val(19); - else if (eq("codec_language_french:")) lang_fr = val(22); - else if (eq("input_ids_len:")) input_ids_len = val(14); - } + if (!kz_text || !kz_vocab_gguf) { + fprintf(stderr, "tts_pipeline: mode fixture (input_ids_full.bin) demandé mais non implémenté\n"); + fprintf(stderr, " -> poser KZTTS_TEXT et KZTTS_VOCAB_GGUF pour utiliser le pipeline live.\n"); + fprintf(stderr, " (Le mode fixture a été déporté en branche fixture pour test_engine séparé.)\n"); + return 2; } - (void)codec_nothink; (void)tts_eos; - printf("manifest: input_ids_len=%d hid=%d text_hid=%d\n", input_ids_len, hidden, text_hidden); - // ----- 1) Charger toutes les fixtures - const double t_load0 = now_s(); - auto text_embed = read_f32(D + "text_embed.bin", (size_t)text_vocab * text_hidden); - auto tp_fc1_w = read_f32(D + "tp_fc1_w.bin", (size_t)text_hidden * text_hidden); - auto tp_fc1_b = read_f32(D + "tp_fc1_b.bin", (size_t)text_hidden); - auto tp_fc2_w = read_f32(D + "tp_fc2_w.bin", (size_t)hidden * text_hidden); - auto tp_fc2_b = read_f32(D + "tp_fc2_b.bin", (size_t)hidden); - auto tok_embd = read_f32(D + "talker_tok_embd.bin", (size_t)3072 * hidden); - auto xvector = read_f32(D + "damien_xvector.bin", (size_t)hidden); + TtsEngineLoadCfg lc; + lc.talker_gguf = talker_gguf; + lc.vocab_gguf = kz_vocab_gguf; + lc.dump_dir = D.c_str(); + lc.use_htp = use_htp; + lc.n_threads = env_i("KZTTS_THREADS", 6); + lc.cp_use_cache = env_i("KZTTS_CP_CACHE", 1) != 0; - // input_ids : depuis le tokenizer C++ (texte arbitraire) ou depuis le dump golden. - // L'engin de l'app initialisera llama_backend_init plus bas dans la section talker ; - // pour pouvoir charger le vocab maintenant, on l'initialise dès ici (idempotent côté llama). - std::vector input_ids; - KzTextTokenizer kz_tok; - if (use_kz_tok) { - // Init backend AVANT chargement vocab. Le talker plus bas ré-init (idempotent côté - // llama). HMX off posé ici par sécurité (le talker le repose ensuite ; sans ça un - // init précoce du backend pourrait verrouiller HMX=on selon l'archi du vocab gguf). - setenv("GGML_HEXAGON_USE_HMX", "0", 1); - llama_backend_init(); - if (!kz_tok_load(kz_tok, kz_vocab_gguf)) { printf("vocab load FAILED: %s\n", kz_vocab_gguf); return 1; } - input_ids = kz_tok_encode_tts_prompt(kz_tok, kz_text); - input_ids_len = (int)input_ids.size(); - if (input_ids_len < 8) { printf("kz_tok: trop court (%d, min=8)\n", input_ids_len); return 1; } - printf("kz_tok: \"%s\" -> %d tokens\n", kz_text, input_ids_len); - } else { - auto v = read_i32(D + "input_ids_full.bin", (size_t)input_ids_len); - input_ids = std::move(v); - } - auto tts_pad_emb_ref = read_f32(D + "tts_pad_embed.bin", (size_t)hidden); // sanity - (void)tts_pad_emb_ref; + auto * eng = tts_engine_load(lc); + if (!eng) { fprintf(stderr, "tts_engine_load FAIL\n"); return 3; } - // ----- 2) Construire prefill_embeds (cf. build_prefill.cpp, validé bit-exact) - const double t_pf0 = now_s(); - std::vector mid(text_hidden); - // special tokens - std::vector spec_text(3 * text_hidden); - int sp[3] = { tts_bos, tts_eos, tts_pad }; - for (int i = 0; i < 3; ++i) - std::memcpy(spec_text.data() + i * text_hidden, text_embed.data() + (size_t)sp[i] * text_hidden, - text_hidden * sizeof(float)); - std::vector spec_proj(3 * hidden); - text_projection(spec_text.data(), 3, tp_fc1_w.data(), tp_fc1_b.data(), - tp_fc2_w.data(), tp_fc2_b.data(), mid.data(), spec_proj.data()); - const float* tts_bos_emb = spec_proj.data() + 0 * hidden; - const float* tts_eos_emb = spec_proj.data() + 1 * hidden; - const float* tts_pad_emb = spec_proj.data() + 2 * hidden; + TtsSynthesizeCfg sc; + sc.text = kz_text; + sc.out_wav_path = out_wav; + sc.max_steps = max_steps; + sc.seed = env_i("KZTTS_SEED", 42); + sc.cp_temp = env_f("KZTTS_CP_TEMP", 0.9f); + sc.cp_top_k = env_i("KZTTS_CP_TOPK", 50); + sc.cp_top_p = env_f("KZTTS_CP_TOPP", 1.0f); + sc.cp_rep_penalty = env_f("KZTTS_CP_REPP", 1.05f); + sc.talker_temp = env_f("KZTTS_TEMP", 0.9f); + sc.talker_top_k = env_i("KZTTS_TOPK", 50); + sc.talker_top_p = env_f("KZTTS_TOPP", 1.0f); + sc.talker_rep_penalty = env_f("KZTTS_REPP", 1.05f); - // codec prefix [think, think_bos, lang_fr, think_eos, x_vector, codec_pad, codec_bos] - int codec_prefill[4] = { codec_think, codec_think_bos, lang_fr, codec_think_eos }; - std::vector codec_input_emb(7 * hidden); - for (int i = 0; i < 4; ++i) - std::memcpy(codec_input_emb.data() + i * hidden, tok_embd.data() + (size_t)codec_prefill[i] * hidden, - hidden * sizeof(float)); - std::memcpy(codec_input_emb.data() + 4 * hidden, xvector.data(), hidden * sizeof(float)); - std::memcpy(codec_input_emb.data() + 5 * hidden, tok_embd.data() + (size_t)codec_pad * hidden, hidden * sizeof(float)); - std::memcpy(codec_input_emb.data() + 6 * hidden, tok_embd.data() + (size_t)codec_bos * hidden, hidden * sizeof(float)); + auto R = tts_engine_synthesize(eng, sc); + if (R.err) { fprintf(stderr, "tts_engine_synthesize FAIL err=%d\n", R.err); tts_engine_free(eng); return 4; } - // role - std::vector role_text(3 * text_hidden); - for (int i = 0; i < 3; ++i) - std::memcpy(role_text.data() + i * text_hidden, text_embed.data() + (size_t)input_ids[i] * text_hidden, - text_hidden * sizeof(float)); - std::vector role_proj(3 * hidden); - text_projection(role_text.data(), 3, tp_fc1_w.data(), tp_fc1_b.data(), - tp_fc2_w.data(), tp_fc2_b.data(), mid.data(), role_proj.data()); + printf("=== TTS : N=%d frames (audio %.2fs) en %.3fs (RTF %.2f) ===\n", + R.frames, R.audio_s, R.total_s, R.total_s / R.audio_s); + printf(" prefill %.3fs | talker_loop %.3fs | cp_loop %.3fs | decoder %.3fs\n", + R.prefill_s, R.talker_loop_s, R.cp_loop_s, R.decoder_s); + printf(" per-frame talker=%.1fms cp=%.1fms\n", + R.talker_loop_s * 1000.0 / R.frames, R.cp_loop_s * 1000.0 / R.frames); + printf("WAV -> %s\n", out_wav); - // text body - const int Nt = input_ids_len - 5 - 3; - std::vector body_text((size_t)Nt * text_hidden); - for (int i = 0; i < Nt; ++i) - std::memcpy(body_text.data() + i * text_hidden, text_embed.data() + (size_t)input_ids[3 + i] * text_hidden, - text_hidden * sizeof(float)); - std::vector body_proj((size_t)Nt * hidden); - text_projection(body_text.data(), Nt, tp_fc1_w.data(), tp_fc1_b.data(), - tp_fc2_w.data(), tp_fc2_b.data(), mid.data(), body_proj.data()); - - // assemble - const int T_prefill = 3 + 6 + (Nt + 1) + 1; - std::vector prefill((size_t)T_prefill * hidden, 0.0f); - std::memcpy(prefill.data(), role_proj.data(), 3 * hidden * sizeof(float)); - for (int i = 0; i < 5; ++i) { - const float* ce = codec_input_emb.data() + i * hidden; - float* p = prefill.data() + (3 + i) * hidden; - for (int d = 0; d < hidden; ++d) p[d] = tts_pad_emb[d] + ce[d]; - } - { const float* ce = codec_input_emb.data() + 5 * hidden; - float* p = prefill.data() + 8 * hidden; - for (int d = 0; d < hidden; ++d) p[d] = tts_bos_emb[d] + ce[d]; } - const float* codec_pad_emb = tok_embd.data() + (size_t)codec_pad * hidden; - for (int i = 0; i < Nt; ++i) { - const float* bp = body_proj.data() + i * hidden; - float* p = prefill.data() + (9 + i) * hidden; - for (int d = 0; d < hidden; ++d) p[d] = bp[d] + codec_pad_emb[d]; - } - { float* p = prefill.data() + (9 + Nt) * hidden; - for (int d = 0; d < hidden; ++d) p[d] = tts_eos_emb[d] + codec_pad_emb[d]; } - { const float* cbe = tok_embd.data() + (size_t)codec_bos * hidden; - float* p = prefill.data() + (10 + Nt) * hidden; - for (int d = 0; d < hidden; ++d) p[d] = tts_pad_emb[d] + cbe[d]; } - printf("prefill constructed T=%d en %.3f s\n", T_prefill, now_s() - t_pf0); - - // ----- 3) Charger talker (engine, libllama) - setenv("GGML_HEXAGON_USE_HMX", "0", 1); - llama_backend_init(); - auto mp = llama_model_default_params(); - ggml_backend_dev_t devs[2] = { force_cpu ? nullptr : find_htp(), nullptr }; - if (devs[0]) { mp.n_gpu_layers = 99; mp.devices = devs; printf("talker: HTP0\n"); } - else { mp.n_gpu_layers = 0; printf("talker: CPU\n"); } - auto m = llama_model_load_from_file(gguf, mp); - if (!m) { printf("talker load FAILED\n"); return 1; } - auto cp = llama_context_default_params(); - cp.n_ctx = std::max(512, T_prefill + max_steps_arg + 16); cp.n_batch = 1024; - cp.n_threads = (getenv("KZTTS_THREADS") ? atoi(getenv("KZTTS_THREADS")) : 6); - cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; - cp.embeddings = true; - auto ctx = llama_init_from_model(m, cp); - if (!ctx) { printf("talker ctx FAILED\n"); return 1; } - const int n_embd = llama_model_n_embd(m); - const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(m)); - if (n_embd != hidden || n_vocab != 3072) { printf("talker dims mismatch (n_embd=%d, n_vocab=%d)\n", n_embd, n_vocab); return 1; } - printf("talker: n_embd=%d n_vocab=%d threads=%d\n", n_embd, n_vocab, cp.n_threads); - - // ----- 4) Charger CP + activer sampling HF-style (rep_penalty + top_k + top_p + temp). - // Sans sampling propre, le greedy tombe dans des attracteurs (codes 690/207/1571 répétés - // = silence/pause) -> WAV avec gros blancs. rep_penalty=1.05 = Python subtalker défaut. - // KZTTS_CP_CACHE=1 -> cp_predict_cached (1 prefill + 14 decode via KV cache, ~8x speedup). - // KZTTS_CP_CACHE=0 (défaut) -> cp_predict (oracle bit-exact, 15 recomputes complets). - const bool cp_use_cache = (getenv("KZTTS_CP_CACHE") && atoi(getenv("KZTTS_CP_CACHE")) != 0); - printf("CP path: %s\n", cp_use_cache ? "CACHED (KZTTS_CP_CACHE=1)" : "RECOMPUTE (oracle bit-exact)"); - CPState cp_state; cp_state.n_threads = cp.n_threads; - cp_state.sampler.temp = getenv("KZTTS_CP_TEMP") ? atof(getenv("KZTTS_CP_TEMP")) : 0.9f; - cp_state.sampler.top_k = getenv("KZTTS_CP_TOPK") ? atoi(getenv("KZTTS_CP_TOPK")) : 50; - cp_state.sampler.top_p = getenv("KZTTS_CP_TOPP") ? atof(getenv("KZTTS_CP_TOPP")) : 1.0f; - cp_state.sampler.rep_penalty = getenv("KZTTS_CP_REPP") ? atof(getenv("KZTTS_CP_REPP")) : 1.05f; - cp_state.sampler.rep_window = 16; - if (!cp_load(cp_state, (D + "cp_f16.gguf").c_str(), (D + "cp_heads.bin").c_str(), (D + "cp_codec_embs.bin").c_str())) { - printf("CP load FAILED\n"); return 1; - } - const uint32_t SEED = getenv("KZTTS_SEED") ? atoi(getenv("KZTTS_SEED")) : 42; - sampler_seed(cp_state.sampler, SEED + 1); // seed différent du Talker pour décorréler - printf("CP sampling: temp=%.2f top_k=%d top_p=%.2f rep_penalty=%.2f\n", - cp_state.sampler.temp, cp_state.sampler.top_k, cp_state.sampler.top_p, cp_state.sampler.rep_penalty); - - // ----- 5) Charger decoder - Decoder dec; - if (!dec.load((D + "qwen3tts_decoder.gguf").c_str())) { printf("decoder load FAILED\n"); return 1; } - - printf("=== load total : %.3f s ===\n", now_s() - t_load0); - - // ----- 6) Pipeline - auto rt = llama_model_rope_type(m); - const int npe = (rt == LLAMA_ROPE_TYPE_MROPE || rt == LLAMA_ROPE_TYPE_IMROPE) ? 4 : 1; - - // --- PREFILL talker - const double t_pfill0 = now_s(); - { - std::vector pos(T_prefill * npe, 0); - std::vector nsd(T_prefill, 1); - std::vector sid0(T_prefill, 0); - std::vector sids(T_prefill); - std::vector lg(T_prefill, 0); - for (int i = 0; i < T_prefill; ++i) { - if (npe == 4) { pos[i] = i; pos[T_prefill + i] = i; pos[2*T_prefill + i] = i; pos[3*T_prefill + i] = 0; } - else { pos[i] = i; } - sids[i] = &sid0[i]; - } - lg[T_prefill - 1] = 1; - llama_batch b{}; - b.n_tokens = T_prefill; b.embd = prefill.data(); - b.pos = pos.data(); b.n_seq_id = nsd.data(); b.seq_id = sids.data(); b.logits = lg.data(); - if (llama_decode(ctx, b) != 0) { printf("prefill FAILED\n"); return 1; } - } - const double t_pfill = now_s() - t_pfill0; - - // --- LOOP - // Sampler Talker (CB0) HF-style. Historique sur les CB0 émis (Talker = phrase entière). - Sampler talker_sampler{}; - talker_sampler.temp = getenv("KZTTS_TEMP") ? (float)atof(getenv("KZTTS_TEMP")) : 0.9f; - talker_sampler.top_k = getenv("KZTTS_TOPK") ? atoi(getenv("KZTTS_TOPK")) : 50; - talker_sampler.top_p = getenv("KZTTS_TOPP") ? (float)atof(getenv("KZTTS_TOPP")) : 1.0f; - talker_sampler.rep_penalty = getenv("KZTTS_REPP") ? (float)atof(getenv("KZTTS_REPP")) : 1.05f; - talker_sampler.rep_window = 64; - sampler_seed(talker_sampler, SEED); - printf("Talker sampling: temp=%.2f top_k=%d top_p=%.2f rep_penalty=%.2f seed=%u\n", - talker_sampler.temp, talker_sampler.top_k, talker_sampler.top_p, talker_sampler.rep_penalty, SEED); - - // logits du prefill : on les copie dans un buffer mutable (le sampler modifie en place) - std::vector logits_buf(n_vocab); - { - const float* lp = llama_get_logits_ith(ctx, -1); - memcpy(logits_buf.data(), lp, n_vocab * sizeof(float)); - } - int cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab); - std::vector hidden_for_cp(n_embd); - { const float* hh = llama_get_embeddings_ith(ctx, -1); if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float)); } - - std::vector codes_engine; - codes_engine.reserve((size_t)max_steps_arg * 16); - int n_eos = -1; - double t_decode_total = 0, t_cp_total = 0; - const double t_loop0 = now_s(); - int N_done = 0; - - for (int s = 0; s < max_steps_arg; ++s) { - // CB0 = greedy - if (cb0 == codec_eos && n_eos < 0) { n_eos = s; printf(" step %d: EOS\n", s); break; } - codes_engine.push_back(cb0); - // CB1..15 via CP - const double tcp0 = now_s(); - const float* cb0_emb = tok_embd.data() + (size_t)cb0 * n_embd; - int32_t cb15[15]; - if (cp_use_cache) cp_predict_cached(cp_state, hidden_for_cp.data(), cb0_emb, cb15); - else cp_predict (cp_state, hidden_for_cp.data(), cb0_emb, cb15); - t_cp_total += now_s() - tcp0; - for (int i = 0; i < 15; ++i) codes_engine.push_back(cb15[i]); - - // next_embed = sum 16 codecs + tts_pad - std::vector next_embed(n_embd, 0.0f); - const float* e_cb0 = tok_embd.data() + (size_t)cb0 * n_embd; - for (int d = 0; d < n_embd; ++d) next_embed[d] = e_cb0[d]; - for (int i = 1; i < 16; ++i) { - int code = cb15[i - 1]; - const float* e = cp_state.codec_embs.data() + ((size_t)(i-1) * 2048 + code) * n_embd; - for (int d = 0; d < n_embd; ++d) next_embed[d] += e[d]; - } - for (int d = 0; d < n_embd; ++d) next_embed[d] += tts_pad_emb[d]; - - // decode talker - llama_pos pos1[4] = {0,0,0,0}; - const llama_pos p = T_prefill + s; - if (npe == 4) { pos1[0] = p; pos1[1] = p; pos1[2] = p; pos1[3] = 0; } - else { pos1[0] = p; } - int32_t nn = 1; llama_seq_id sd = 0; llama_seq_id* sp = &sd; int8_t l = 1; - llama_batch b{}; - b.n_tokens = 1; b.embd = next_embed.data(); - b.pos = pos1; b.n_seq_id = &nn; b.seq_id = &sp; b.logits = &l; - const double td0 = now_s(); - if (llama_decode(ctx, b) != 0) { printf("step %d FAILED\n", s); break; } - t_decode_total += now_s() - td0; - - { - const float* lp = llama_get_logits_ith(ctx, -1); - memcpy(logits_buf.data(), lp, n_vocab * sizeof(float)); - } - cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab); - const float* hh = llama_get_embeddings_ith(ctx, -1); - if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float)); - N_done = s + 1; - } - const double t_loop = now_s() - t_loop0; - const int N = N_done; - const double audio_s = N / 12.0; - printf("=== TTS Talker+CP : N=%d frames (audio %.2fs) en %.3fs (RTF %.2f) ===\n", - N, audio_s, t_pfill + t_loop, (t_pfill + t_loop) / audio_s); - printf(" prefill %.3fs | loop %.3fs (talker=%.3f cp=%.3f)\n", t_pfill, t_loop, t_decode_total, t_cp_total); - printf(" per-step talker=%.1fms cp=%.1fms\n", t_decode_total * 1000.0 / N, t_cp_total * 1000.0 / N); - - // dump codes for debug - { - std::ofstream f("/data/local/tmp/kz-engine/pipeline_codes.bin", std::ios::binary); - f.write((const char*)codes_engine.data(), codes_engine.size() * sizeof(int32_t)); - } - printf("codes (CB0 trajectory only, %d frames):\n ", N); - for (int t = 0; t < N; ++t) { - printf("%d ", codes_engine[t * 16 + 0]); - if ((t + 1) % 16 == 0) printf("\n "); - } - printf("\n"); - // ----- 7) Decoder ggml : codes [N, 16] (time-major) -> WAV - // Decoder veut codes_flat[16 * T] codebook-major (CB-fastest dans son forward). - // codes_engine = [N, 16] time-major -> transpose en [16, N] codebook-major. - std::vector codes_dec(16 * N); - for (int t = 0; t < N; ++t) - for (int c = 0; c < 16; ++c) - codes_dec[c * N + t] = codes_engine[t * 16 + c]; - - const double t_dec0 = now_s(); - auto wav = dec.forward(codes_dec, N); - const double t_dec = now_s() - t_dec0; - printf("=== Decoder : %.3fs (RTF dec %.2f) -> %zu samples (%.2fs @24k)\n", - t_dec, t_dec / audio_s, wav.size(), wav.size() / 24000.0); - - write_wav_pcm16_mono(OUT_WAV, wav.data(), wav.size(), 24000); - printf("=== TOTAL pipeline : %.3fs -> RTF %.2f ===\n", - t_pfill + t_loop + t_dec, (t_pfill + t_loop + t_dec) / audio_s); - printf("WAV -> %s\n", OUT_WAV); - - cp_free(cp_state); - llama_free(ctx); - llama_model_free(m); - if (use_kz_tok) kz_tok_free(kz_tok); + tts_engine_free(eng); return 0; }