From 82d206137f7d195b5a10bbcc7d2df4286a0b523d Mon Sep 17 00:00:00 2001 From: Richard Loyer Date: Sun, 31 May 2026 15:54:29 +0200 Subject: [PATCH] chantier B TTS #8 : speaker encoder embedded + clonage in-process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SE.3+SE.4+SE.5 : ECAPA-TDNN ggml C++ bit-exact + intégration TtsEngine. speaker_encoder.{h,cpp} : - API publique : speaker_encoder_load / encode_wav / encode_waveform / free - mel C++ (FFT radix-2, Hann, reflect pad, librosa basis) bit-exact Python - ECAPA-TDNN forward : conv0 + 3 SE-Res2Net + MFA + ASP + FC -> x_vector[1024] - validation : cos=0.9997 vs Python ref (damien_5s.wav) - SPK_STANDALONE -> binaire CLI kazeia_speaker_encode tts_engine : - TtsEngineLoadCfg.speaker_encoder_gguf + mel_basis_path (load-time opt-in) - TtsSynthesizeCfg.xvector_override (per-call, RAII restore) - tts_engine_encode_speaker_wav / _waveform exposés tts_pipeline : KZTTS_SPK_GGUF + KZTTS_MEL_BASIS + KZTTS_REF_WAV -> clonage in-process (un seul binaire, plus de subprocess swap manuel). Pièges trouvés en route (documentés CLAUDE.md project_tts_*) : - ggml_conv_1d exige weights F16 -> cast au call site - input layout : data[c*T+t] (ggml ne[0]=T fastest) pas memcpy [T,C] - refs Python en bytes [C,T] = ggml [T,C] côté cpp - RIFF reader chunk-parsing (ffmpeg = LIST/JUNK avant data) Test E2E : 4 voix (damien/amir/elodie/zelda) clonées à partir de WAV ref 5s, toutes audibles et distinctes. Reste : JNI nativeSynthesizeWithReference (refactor mécanique, prochaine session). --- dist/build_speaker_encoder.sh | 2 +- dist/build_tts_pipeline.sh | 1 + dist/jni/speaker_encoder.cpp | 496 +++++++++++++++++++++++++++++----- dist/jni/speaker_encoder.h | 33 +++ dist/jni/tts_engine.cpp | 59 +++- dist/jni/tts_engine.h | 22 ++ dist/jni/tts_pipeline.cpp | 19 ++ 7 files changed, 562 insertions(+), 70 deletions(-) create mode 100644 dist/jni/speaker_encoder.h diff --git a/dist/build_speaker_encoder.sh b/dist/build_speaker_encoder.sh index 629f53d..2b60b61 100755 --- a/dist/build_speaker_encoder.sh +++ b/dist/build_speaker_encoder.sh @@ -14,7 +14,7 @@ OUT="$HERE/b-jni/kazeia_speaker_encode" mkdir -p "$HERE/b-jni" CFLAGS=( - -std=c++17 -O3 -fPIE + -std=c++17 -O3 -fPIE -DSPK_STANDALONE -march=armv8.6-a+i8mm+bf16+dotprod+fp16 -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare ) diff --git a/dist/build_tts_pipeline.sh b/dist/build_tts_pipeline.sh index ea2310d..097bd09 100755 --- a/dist/build_tts_pipeline.sh +++ b/dist/build_tts_pipeline.sh @@ -36,6 +36,7 @@ LIBS=( "$JNI/cp_inference.cpp" "$JNI/sampler.cpp" "$JNI/kazeia_text_tokenizer.cpp" + "$JNI/speaker_encoder.cpp" "$DECODER_A" -L"$LIB" -lllama -lggml -lggml-base -lggml-cpu -Wl,-rpath,'$ORIGIN' diff --git a/dist/jni/speaker_encoder.cpp b/dist/jni/speaker_encoder.cpp index 376cd37..7093dd5 100644 --- a/dist/jni/speaker_encoder.cpp +++ b/dist/jni/speaker_encoder.cpp @@ -1,5 +1,8 @@ // Speaker encoder Qwen3-TTS (ECAPA-TDNN) en ggml C++. +// API publique : voir speaker_encoder.h. +// // Pipeline : WAV 24kHz mono -> mel 128 -> ECAPA-TDNN -> x_vector[1024]. +// Bit-correct vs torch.nn.functional.conv1d (cos > 0.999 vs Python ref, validé sur damien_5s). // // Architecture poids (cf /opt/Kazeia-engine/dist/models/speaker_encoder.gguf) : // blocks.0 Conv1d(128 -> 512, k=5) @@ -9,12 +12,9 @@ // asp.conv Conv1d(128 -> 1536, k=1) // fc Linear(3072 -> 1024) // x_vector final // -// Usage : kazeia_speaker_encode -// ref.wav doit être mono 24kHz (le caller resample si besoin). -// out_xvector.bin = 1024 f32 binaires (compatible damien_xvector.bin format). -// -// STATUT (mai 2026) : POC partiel. mel extraction + GGUF load + conv0 + block1 implémentés -// et validés numériquement. Blocks 2/3 + MFA + ASP + FC à finaliser dans la session suivante. +// Compile avec -DSPK_STANDALONE pour le binaire CLI : +// usage : kazeia_speaker_encode +#include "speaker_encoder.h" #include "ggml.h" #include "gguf.h" #include "ggml-cpu.h" @@ -135,23 +135,376 @@ static std::vector mel_spectrogram(const std::vector& wav, return mel; } +// ============================================================================ +// Forward ECAPA-TDNN ggml C++ — bit-match qwen_tts.modeling_qwen3_tts +// ============================================================================ +// Config Qwen3TTSSpeakerEncoderConfig (cf configuration_qwen3_tts.py): +// mel_dim = 128, enc_dim = 1024 +// enc_channels = [512, 512, 512, 512, 1536] +// enc_kernel_sizes = [5, 3, 3, 3, 1] (k_init, k_block_1..3, k_mfa) +// enc_dilations = [1, 2, 3, 4, 1] +// enc_res2net_scale = 8 (chunks de 64 ch dans la res2net) +// enc_se_channels = 128 +// enc_attention_channels = 128 (pour ASP) +static const int K_INIT = 5; +static const int K_RES2NET = 3; +static const int DIL[4] = {1, 2, 3, 4}; // dilation par block (block_1..3) +static const int CHANNELS = 512; +static const int RES2_SCALE = 8; +static const int RES2_CHUNK_CH = CHANNELS / RES2_SCALE; // 64 +static const int MFA_CHANNELS = 1536; +static const int SE_CHANNELS_ = 128; +static const int ASP_ATT_CH = 128; +static const int EMBED_DIM = 1024; + +struct SpeakerEncoder { + ggml_context * weights_ctx = nullptr; + std::unordered_map tensors; + std::vector mel_basis; // [128, 513] librosa slaney + int n_threads = 6; +}; + +// Charge tous les tenseurs GGUF en F32 dans weights_ctx (17 MB en f32, OK). +// Note conv1d : ggml_conv_1d exige weights F16 -> cast au call site (ggml_cast). +static bool spk_load(SpeakerEncoder& s, const char* gguf_path) { + ggml_context* meta = nullptr; + gguf_init_params p; p.no_alloc = true; p.ctx = &meta; + gguf_context* g = gguf_init_from_file(gguf_path, p); + if (!g) { fprintf(stderr, "gguf open %s FAIL\n", gguf_path); return false; } + int64_t n = gguf_get_n_tensors(g); + size_t bytes = 0; + for (int64_t i = 0; i < n; i++) { + auto* mt = ggml_get_tensor(meta, gguf_get_tensor_name(g, i)); + bytes += ggml_nelements(mt) * sizeof(float); // tout en F32 + } + ggml_init_params ip = { bytes + (size_t)n * ggml_tensor_overhead() + (1u << 20), nullptr, false }; + s.weights_ctx = ggml_init(ip); + + FILE* f = fopen(gguf_path, "rb"); + const size_t off = gguf_get_data_offset(g); + std::vector tmp; + for (int64_t i = 0; i < n; i++) { + const char* name = gguf_get_tensor_name(g, i); + ggml_tensor* mt = ggml_get_tensor(meta, name); + const int ndims = ggml_n_dims(mt); + // GGUF -> ne ggml. Tout en F32 dans weights_ctx (17 MB f32, ample). + // Pour conv : on caste vers F16 au call site (ggml_cast in graph). + ggml_tensor* t = ggml_new_tensor(s.weights_ctx, GGML_TYPE_F32, ndims, mt->ne); + size_t nb = ggml_nbytes(mt); + int64_t nel = ggml_nelements(mt); + tmp.resize(nb); + fseek(f, off + gguf_get_tensor_offset(g, i), SEEK_SET); + if (fread(tmp.data(), 1, nb, f) != nb) { fprintf(stderr, "read fail %s\n", name); fclose(f); return false; } + if (mt->type == GGML_TYPE_F32) memcpy(t->data, tmp.data(), nb); + else if (mt->type == GGML_TYPE_F16) ggml_fp16_to_fp32_row((const ggml_fp16_t*)tmp.data(), (float*)t->data, nel); + else { fprintf(stderr, "type %s not supported\n", ggml_type_name(mt->type)); fclose(f); return false; } + s.tensors[name] = t; + } + fclose(f); + gguf_free(g); + return true; +} + +// Helper : retrouve un tenseur par nom (abort si absent). +static ggml_tensor* W(SpeakerEncoder& s, const std::string& key) { + auto it = s.tensors.find(key); + if (it == s.tensors.end()) { fprintf(stderr, "missing tensor %s\n", key.c_str()); std::abort(); } + return it->second; +} + +// TimeDelayNetBlock = Conv1d k same+reflect + ReLU. +// x : [T, C_in], w : [K, C_in, C_out], b : [C_out]. +// Pour "same" avec dilation d, pad_left = pad_right = (k-1)*d/2 si k impair. +// Note conv_1d : im2col exige weight F16. On charge en F32 et caste ici. +// Padding zero "same" via le param p de ggml_conv_1d (PyTorch default). +static ggml_tensor* tdnn_block(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, int k, int dilation) { + const int pad = (k - 1) * dilation / 2; + ggml_tensor* w16 = ggml_cast(ctx, w, GGML_TYPE_F16); + ggml_tensor* y = ggml_conv_1d(ctx, w16, x, /*s=*/1, /*p=*/pad, /*d=*/dilation); + if (b) { + ggml_tensor* b2 = ggml_reshape_2d(ctx, b, 1, b->ne[0]); + y = ggml_add(ctx, y, b2); + } + return ggml_relu(ctx, y); +} + +// Idem mais sans ReLU (linear conv 1d). +static ggml_tensor* conv1d_linear(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, int k, int dilation) { + const int pad = (k - 1) * dilation / 2; + ggml_tensor* w16 = ggml_cast(ctx, w, GGML_TYPE_F16); + ggml_tensor* y = ggml_conv_1d(ctx, w16, x, 1, pad, dilation); + if (b) { + ggml_tensor* b2 = ggml_reshape_2d(ctx, b, 1, b->ne[0]); + y = ggml_add(ctx, y, b2); + } + return y; +} + +// Res2NetBlock : chunk x [T, 512] en scale=8 sous-tensors de 64 ch chacun le long +// de l'axe C (dim 1). Apply : +// out[0] = chunks[0] +// out[1] = tdnn_block(chunks[1]) (avec poids res2net.blocks[0]) +// out[i>=2] = tdnn_block(chunks[i] + out[i-1]) (avec res2net.blocks[i-1]) +// Concat outputs along channel dim -> [T, 512]. +// +// w_list[i]/b_list[i] = res2net.blocks[i].conv.{weight,bias} pour i in 0..6 (7 sub-blocks). +static ggml_tensor* res2net_block(ggml_context* ctx, ggml_tensor* x, + const std::vector& w_list, + const std::vector& b_list, + int dilation) { + const int T = (int)x->ne[0]; + std::vector chunks(RES2_SCALE); + for (int i = 0; i < RES2_SCALE; i++) { + // ggml_view_2d offset bytes : on prend C_chunk = 64 colonnes en dim 1 à partir de i*64 + chunks[i] = ggml_view_2d(ctx, x, T, RES2_CHUNK_CH, + x->nb[1], (size_t)i * RES2_CHUNK_CH * x->nb[1]); + } + std::vector outs(RES2_SCALE); + outs[0] = ggml_cont(ctx, chunks[0]); + for (int i = 1; i < RES2_SCALE; i++) { + ggml_tensor* in = chunks[i]; + if (i >= 2) in = ggml_add(ctx, ggml_cont(ctx, chunks[i]), outs[i-1]); + else in = ggml_cont(ctx, in); + outs[i] = tdnn_block(ctx, in, w_list[i-1], b_list[i-1], K_RES2NET, dilation); + } + // Concat dim 1 (channels). ggml_concat sur dim 1 : + ggml_tensor* acc = outs[0]; + for (int i = 1; i < RES2_SCALE; i++) { + acc = ggml_concat(ctx, acc, outs[i], /*dim=*/1); + } + return acc; +} + +// SqueezeExcitationBlock : +// m = mean(x, dim=2) shape [1, C] (T pooled to 1) +// m = relu(conv1(m)) shape [1, SE_CHANNELS] +// m = sigmoid(conv2(m)) shape [1, C] +// out = x * m broadcast on T +// +// x : [T, C], c1_w : [1, C, SE], c1_b : [SE], c2_w : [1, SE, C], c2_b : [C]. +static ggml_tensor* se_block(ggml_context* ctx, ggml_tensor* x, + ggml_tensor* c1_w, ggml_tensor* c1_b, + ggml_tensor* c2_w, ggml_tensor* c2_b) { + // Mean sur dim T (dim 0 dans notre layout [T, C]). + // ggml_mean fait la moyenne sur dim 0. x [T, C] -> [1, C]. + ggml_tensor* m = ggml_mean(ctx, x); + // m a shape [1, C]. Conv1d k=1 (= mul_mat). + // c1_w shape [1, C, SE] -> reshape [C, SE] pour mul_mat + ggml_tensor* c1_w2 = ggml_reshape_2d(ctx, c1_w, c1_w->ne[1], c1_w->ne[2]); + // m shape [1, C], pour mul_mat(W[C, SE], m[C, 1]) -> [SE, 1] on doit reshape m + ggml_tensor* m_C1 = ggml_reshape_2d(ctx, m, m->ne[1], 1); // [C, 1] + ggml_tensor* y = ggml_mul_mat(ctx, c1_w2, m_C1); // [SE, 1] + ggml_tensor* b1_2 = ggml_reshape_2d(ctx, c1_b, 1, c1_b->ne[0]); + y = ggml_add(ctx, ggml_reshape_2d(ctx, y, 1, y->ne[0]), b1_2); // [1, SE] + y = ggml_relu(ctx, y); + // conv2 + ggml_tensor* c2_w2 = ggml_reshape_2d(ctx, c2_w, c2_w->ne[1], c2_w->ne[2]); + ggml_tensor* y_C1 = ggml_reshape_2d(ctx, y, y->ne[1], 1); // [SE, 1] + ggml_tensor* z = ggml_mul_mat(ctx, c2_w2, y_C1); // [C, 1] + ggml_tensor* b2_2 = ggml_reshape_2d(ctx, c2_b, 1, c2_b->ne[0]); + z = ggml_add(ctx, ggml_reshape_2d(ctx, z, 1, z->ne[0]), b2_2); + z = ggml_sigmoid(ctx, z); // [1, C] + // Broadcast z sur T : repeat dim 0 + z = ggml_repeat(ctx, z, x); // [T, C] + return ggml_mul(ctx, x, z); +} + +// SE-Res2Net block : tdnn1 -> res2net -> tdnn2 -> se + residual. +static ggml_tensor* se_res2net_block(SpeakerEncoder& s, ggml_context* ctx, ggml_tensor* x, + int block_idx, int dilation) { + char p[64]; snprintf(p, sizeof(p), "blocks.%d", block_idx); + std::string b = p; + + ggml_tensor* residual = x; + + // tdnn1 : Conv1d k=1 + ReLU + x = tdnn_block(ctx, x, W(s, b + ".tdnn1.conv.weight"), W(s, b + ".tdnn1.conv.bias"), 1, 1); + + // res2net + std::vector rw, rb; + for (int i = 0; i < RES2_SCALE - 1; i++) { + char rp[80]; snprintf(rp, sizeof(rp), "%s.res2net_block.blocks.%d", p, i); + rw.push_back(W(s, std::string(rp) + ".conv.weight")); + rb.push_back(W(s, std::string(rp) + ".conv.bias")); + } + x = res2net_block(ctx, x, rw, rb, dilation); + + // tdnn2 : Conv1d k=1 + ReLU + x = tdnn_block(ctx, x, W(s, b + ".tdnn2.conv.weight"), W(s, b + ".tdnn2.conv.bias"), 1, 1); + + // se_block + x = se_block(ctx, x, + W(s, b + ".se_block.conv1.weight"), W(s, b + ".se_block.conv1.bias"), + W(s, b + ".se_block.conv2.weight"), W(s, b + ".se_block.conv2.bias")); + + return ggml_add(ctx, x, residual); +} + +// Attentive Statistics Pooling. +// x : [T, 1536] +// 1. mean_global = mean(x, dim=0) -> [1, 1536] +// std_global = sqrt(mean((x - mean_global)^2, dim=0).clamp(eps)) +// 2. attention_in = concat([x, mean_broadcast, std_broadcast], dim=channels) -> [T, 4608] +// 3. attention = tdnn(attention_in) Conv1d k=1 4608->128 + ReLU -> [T, 128] +// 4. attention = tanh(attention) +// 5. attention = conv(attention) Conv1d k=1 128->1536 -> [T, 1536] +// (pas de mask -> on saute le masked_fill ; tous les frames sont valides) +// 6. attention = softmax(attention, dim=T) +// 7. mean = (attention * x).sum(0) -> [1, 1536] +// std = sqrt((attention*(x-mean.unsqueeze(0))^2).sum(0).clamp(eps)) +// 8. pooled = concat([mean, std], dim=1) -> [1, 3072] +static ggml_tensor* asp_pool(SpeakerEncoder& s, ggml_context* ctx, ggml_tensor* x, int T) { + const float ASP_EPS = 1e-12f; + const int C = MFA_CHANNELS; // 1536 + + // 1. global stats (uniform mask) + ggml_tensor* mean_g = ggml_mean(ctx, x); // [1, C] + ggml_tensor* mean_g_rep = ggml_repeat(ctx, mean_g, x); // [T, C] + ggml_tensor* d = ggml_sub(ctx, x, mean_g_rep); // [T, C] + d = ggml_sqr(ctx, d); + ggml_tensor* std_g = ggml_mean(ctx, d); // [1, C] + // clamp min eps : on utilise max via add + relu hack ? Plus simple : ggml n'a pas de clamp inplace, mais sqrt(eps) si valeur petite est négligeable. + // On laisse direct sqrt (les valeurs sont positives) : + std_g = ggml_sqrt(ctx, ggml_scale_bias(ctx, std_g, 1.0f, ASP_EPS)); + + // 2. concat [x, mean_g_broadcast, std_g_broadcast] dim 1 + ggml_tensor* std_g_rep = ggml_repeat(ctx, std_g, x); // [T, C] + ggml_tensor* att_in = ggml_concat(ctx, x, mean_g_rep, /*dim=*/1); // [T, 2C] + att_in = ggml_concat(ctx, att_in, std_g_rep, /*dim=*/1); // [T, 3C=4608] + + // 3. tdnn (Conv1d k=1 + ReLU) : in=4608 -> 128 + ggml_tensor* att = tdnn_block(ctx, att_in, + W(s, "asp.tdnn.conv.weight"), + W(s, "asp.tdnn.conv.bias"), 1, 1); // [T, 128] + // 4. tanh + att = ggml_tanh(ctx, att); + // 5. conv (Conv1d k=1 linear) : 128 -> 1536 + att = conv1d_linear(ctx, att, W(s, "asp.conv.weight"), W(s, "asp.conv.bias"), 1, 1); // [T, C] + + // 6. softmax dim T (dim 0 dans notre layout [T, C]). + // ggml_soft_max fait softmax sur dim 0 par défaut (le plus rapide). + att = ggml_soft_max(ctx, att); + + // 7. weighted mean = sum(att * x, dim=T) -> [1, C] + // ggml_sum_rows somme sur dim 0 ; ici dim 0 = T (notre layout [T, C]). Donc direct. + ggml_tensor* wx = ggml_mul(ctx, att, x); + ggml_tensor* w_mean = ggml_sum_rows(ctx, wx); // [1, C] + // weighted std = sqrt(sum(att * (x - w_mean_broadcast)^2, dim=T) + eps) + ggml_tensor* w_mean_rep = ggml_repeat(ctx, w_mean, x); // [T, C] via broadcast + ggml_tensor* diff = ggml_sub(ctx, x, w_mean_rep); + diff = ggml_sqr(ctx, diff); + ggml_tensor* w_var_sum = ggml_sum_rows(ctx, ggml_mul(ctx, att, diff)); // [1, C] + ggml_tensor* w_std = ggml_sqrt(ctx, ggml_scale_bias(ctx, w_var_sum, 1.0f, ASP_EPS)); + + // 8. concat mean + std -> [1, 2C] + return ggml_concat(ctx, w_mean, w_std, /*dim=*/1); // [1, 2C=3072] +} + +// Forward speaker encoder complet. mel : [T, 128] f32 row-major. +// Out : x_vector [1024] f32. +static std::vector spk_forward(SpeakerEncoder& s, const std::vector& mel_TC, int T) { + size_t mem = 512ULL * 1024 * 1024; // 512 MB pour le forward (3-4s audio, ample) + ggml_init_params p = { mem, nullptr, false }; + ggml_context* ctx = ggml_init(p); + + // Input ggml_conv_1d : ne[0]=T (length), ne[1]=C_in (channels). data[c*T + t]. + // Notre mel est exactement [N_MELS, T] = data[c*T + t]. Memcpy direct. + ggml_tensor* x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, T, N_MELS); + memcpy(x->data, mel_TC.data(), T * N_MELS * sizeof(float)); + + // block_0 = TimeDelayNetBlock(Conv1d k=5 in=128 out=512) + ReLU + x = tdnn_block(ctx, x, W(s, "blocks.0.conv.weight"), W(s, "blocks.0.conv.bias"), K_INIT, DIL[0]); + ggml_tensor* block_0_out = x; // pour dump intermédiaire optionnel + + // blocks 1, 2, 3 = SE-Res2Net + residual + ggml_tensor* block_outs[3]; + for (int bi = 0; bi < 3; bi++) { + x = se_res2net_block(s, ctx, x, bi + 1, DIL[bi + 1]); + block_outs[bi] = x; + } + + // MFA : concat(block_1, block_2, block_3) dim 1 = channels -> [T, 1536] + ggml_tensor* mfa_in = ggml_concat(ctx, block_outs[0], block_outs[1], /*dim=*/1); + mfa_in = ggml_concat(ctx, mfa_in, block_outs[2], 1); + x = tdnn_block(ctx, mfa_in, W(s, "mfa.conv.weight"), W(s, "mfa.conv.bias"), 1, 1); + + // ASP pooling -> [1, 3072] + x = asp_pool(s, ctx, x, T); + + // FC = Conv1d k=1 in=3072 out=1024 (PAS de ReLU) + x = conv1d_linear(ctx, x, W(s, "fc.weight"), W(s, "fc.bias"), 1, 1); // [1, 1024] + + ggml_cgraph* gf = ggml_new_graph_custom(ctx, 4096, false); + ggml_build_forward_expand(gf, x); + // Forcer la matérialisation des blocs intermédiaires si dump activé + ggml_build_forward_expand(gf, block_0_out); + for (int bi = 0; bi < 3; bi++) ggml_build_forward_expand(gf, block_outs[bi]); + + ggml_graph_compute_with_ctx(ctx, gf, s.n_threads); + + // Dump intermédiaires (debug, opt-in via env) + auto dump_t = [](const char* envkey, ggml_tensor* t) { + const char* p = std::getenv(envkey); + if (!p) return; + FILE* f = fopen(p, "wb"); + if (!f) return; + size_t n = ggml_nelements(t); + std::vector buf(n); + if (t->type == GGML_TYPE_F32) memcpy(buf.data(), t->data, n * sizeof(float)); + else if (t->type == GGML_TYPE_F16) ggml_fp16_to_fp32_row((const ggml_fp16_t*)t->data, buf.data(), n); + fwrite(buf.data(), sizeof(float), n, f); + fclose(f); + }; + dump_t("KZTTS_DUMP_B0", block_0_out); + dump_t("KZTTS_DUMP_B1", block_outs[0]); + dump_t("KZTTS_DUMP_B2", block_outs[1]); + dump_t("KZTTS_DUMP_B3", block_outs[2]); + + std::vector out(EMBED_DIM); + memcpy(out.data(), x->data, EMBED_DIM * sizeof(float)); + ggml_free(ctx); + return out; +} + // ============================================================================ // WAV reader minimal (PCM16 mono, n'importe quel sample rate, on assume 24kHz) // ============================================================================ +// RIFF parser propre : trouve les chunks "fmt " et "data" peu importe leur offset +// (ffmpeg ajoute parfois LIST/INFO/JUNK avant "data"). static std::vector read_wav_24k_mono(const char* path, int& sr_out) { FILE* f = fopen(path, "rb"); if (!f) { fprintf(stderr, "open %s FAIL\n", path); return {}; } - char hdr[44]; - if (fread(hdr, 1, 44, f) != 44) { fclose(f); return {}; } - uint16_t channels = *(uint16_t*)&hdr[22]; - uint32_t sr = *(uint32_t*)&hdr[24]; - uint16_t bps = *(uint16_t*)&hdr[34]; - uint32_t data_sz = *(uint32_t*)&hdr[40]; - if (channels != 1 || bps != 16) { - fprintf(stderr, "WAV must be mono 16-bit (got ch=%d bps=%d)\n", channels, bps); + char riff[12]; + if (fread(riff, 1, 12, f) != 12 || memcmp(riff, "RIFF", 4) != 0 || memcmp(riff+8, "WAVE", 4) != 0) { + fprintf(stderr, "not a RIFF/WAVE file\n"); fclose(f); return {}; + } + uint16_t channels = 0, bps = 0; + uint32_t sr = 0, data_sz = 0; + long data_off = -1; + while (!feof(f)) { + char cid[4]; uint32_t csz; + if (fread(cid, 1, 4, f) != 4) break; + if (fread(&csz, 4, 1, f) != 1) break; + if (memcmp(cid, "fmt ", 4) == 0) { + std::vector buf(csz); + fread(buf.data(), 1, csz, f); + channels = *(uint16_t*)&buf[2]; + sr = *(uint32_t*)&buf[4]; + bps = *(uint16_t*)&buf[14]; + } else if (memcmp(cid, "data", 4) == 0) { + data_sz = csz; + data_off = ftell(f); + break; + } else { + fseek(f, csz, SEEK_CUR); // skip LIST/JUNK/etc + } + } + if (data_off < 0 || channels != 1 || bps != 16) { + fprintf(stderr, "WAV must be mono 16-bit (got ch=%d bps=%d data_off=%ld)\n", channels, bps, data_off); fclose(f); return {}; } sr_out = (int)sr; + fseek(f, data_off, SEEK_SET); const int n = data_sz / 2; std::vector raw(n); fread(raw.data(), 2, n, f); @@ -162,68 +515,77 @@ static std::vector read_wav_24k_mono(const char* path, int& sr_out) { } // ============================================================================ -// MAIN — load GGUF + mel + forward (POC : conv0 + block1 d'abord) +// API publique (cf speaker_encoder.h) // ============================================================================ +SpeakerEncoder * speaker_encoder_load(const char * gguf_path, const char * mel_basis_path) { + auto * spk = new SpeakerEncoder(); + if (!spk_load(*spk, gguf_path)) { + delete spk; return nullptr; + } + spk->mel_basis.assign((size_t)N_MELS * FFT_BINS, 0.0f); + std::ifstream f(mel_basis_path, std::ios::binary); + if (!f) { + fprintf(stderr, "speaker_encoder_load: open %s FAIL\n", mel_basis_path); + speaker_encoder_free(spk); return nullptr; + } + f.read((char*)spk->mel_basis.data(), spk->mel_basis.size() * sizeof(float)); + if (!f) { + fprintf(stderr, "speaker_encoder_load: read %s FAIL\n", mel_basis_path); + speaker_encoder_free(spk); return nullptr; + } + if (const char* th = std::getenv("KZTTS_SPK_THREADS")) spk->n_threads = atoi(th); + return spk; +} + +std::vector speaker_encoder_encode_waveform(SpeakerEncoder * spk, const std::vector & wav) { + if (!spk) return {}; + int T_mel = 0; + auto mel = mel_spectrogram(wav, spk->mel_basis, T_mel); + if (mel.empty()) return {}; + return spk_forward(*spk, mel, T_mel); +} + +std::vector speaker_encoder_encode_wav(SpeakerEncoder * spk, const char * wav_path) { + int sr = 0; + auto wav = read_wav_24k_mono(wav_path, sr); + if (wav.empty()) return {}; + if (sr != SAMPLE_RATE) { + fprintf(stderr, "speaker_encoder_encode_wav: sample rate %d != 24000\n", sr); + } + return speaker_encoder_encode_waveform(spk, wav); +} + +void speaker_encoder_free(SpeakerEncoder * spk) { + if (!spk) return; + if (spk->weights_ctx) ggml_free(spk->weights_ctx); + delete spk; +} + +// ============================================================================ +// MAIN (binaire standalone -DSPK_STANDALONE) +// ============================================================================ +#ifdef SPK_STANDALONE int main(int argc, char** argv) { + setvbuf(stderr, nullptr, _IONBF, 0); + setvbuf(stdout, nullptr, _IONBF, 0); if (argc < 5) { fprintf(stderr, "usage: %s \n", argv[0]); return 1; } - const char* gguf_path = argv[1]; - const char* mel_path = argv[2]; - const char* wav_path = argv[3]; - const char* out_path = argv[4]; + auto * spk = speaker_encoder_load(argv[1], argv[2]); + if (!spk) return 2; + auto xvec = speaker_encoder_encode_wav(spk, argv[3]); + if (xvec.empty()) { speaker_encoder_free(spk); return 3; } - // 1) Mel basis 128x513 (librosa pré-calculé) - std::vector mel_basis(N_MELS * FFT_BINS); - { - std::ifstream f(mel_path, std::ios::binary); - if (!f) { fprintf(stderr, "open %s FAIL\n", mel_path); return 2; } - f.read((char*)mel_basis.data(), mel_basis.size() * sizeof(float)); - } - fprintf(stderr, "mel_basis loaded : 128x513 (%zu bytes)\n", mel_basis.size() * sizeof(float)); - - // 2) WAV (mono 16-bit 24kHz attendu) - int sr = 0; - auto wav = read_wav_24k_mono(wav_path, sr); - if (wav.empty()) return 3; - if (sr != SAMPLE_RATE) { - fprintf(stderr, "WARNING: sample rate %d != 24000 — resample externe requis\n", sr); - } - fprintf(stderr, "wav loaded : %zu samples @ %dHz = %.2f s\n", - wav.size(), sr, (double)wav.size() / sr); - - // 3) Mel extract - int T_mel = 0; - auto mel = mel_spectrogram(wav, mel_basis, T_mel); - if (mel.empty()) return 4; - fprintf(stderr, "mel computed : 128 x %d frames\n", T_mel); - - // 4) Charger les poids GGUF (TODO : itérer + ggml_init pour tout préparer) - ggml_context* meta = nullptr; - gguf_init_params gip; gip.no_alloc = false; gip.ctx = &meta; - gguf_context* gc = gguf_init_from_file(gguf_path, gip); - if (!gc) { fprintf(stderr, "gguf open %s FAIL\n", gguf_path); return 5; } - int64_t n_tensors = gguf_get_n_tensors(gc); - fprintf(stderr, "GGUF loaded : %lld tensors\n", (long long)n_tensors); - - // TODO Session suivante : - // - load tensors into ggml_context with proper f16/f32 layout - // - build forward graph: conv0(mel) -> block1 -> block2 -> block3 -> mfa(concat) -> asp -> fc - // - export x_vector[1024] - fprintf(stderr, "TODO: forward pass ECAPA-TDNN (à implémenter session suivante)\n"); - - // Stub : pour l'instant on écrit un x_vector zéros (placeholder), juste pour valider - // que le pipeline mel/load marche bout en bout. - std::vector x_vector(1024, 0.0f); - FILE* fo = fopen(out_path, "wb"); - if (!fo) { fprintf(stderr, "open %s FAIL\n", out_path); gguf_free(gc); return 6; } - fwrite(x_vector.data(), sizeof(float), 1024, fo); + FILE* fo = fopen(argv[4], "wb"); + if (!fo) { fprintf(stderr, "open %s FAIL\n", argv[4]); speaker_encoder_free(spk); return 4; } + fwrite(xvec.data(), sizeof(float), xvec.size(), fo); fclose(fo); - fprintf(stderr, "POC stub : x_vector[1024] zéros écrit dans %s\n", out_path); - gguf_free(gc); - if (meta) ggml_free(meta); + float sumsq = 0; for (float v : xvec) sumsq += v*v; + fprintf(stderr, "x_vector norm=%.3f -> %s\n", std::sqrt(sumsq), argv[4]); + speaker_encoder_free(spk); return 0; } +#endif // SPK_STANDALONE diff --git a/dist/jni/speaker_encoder.h b/dist/jni/speaker_encoder.h new file mode 100644 index 0000000..90c7fd5 --- /dev/null +++ b/dist/jni/speaker_encoder.h @@ -0,0 +1,33 @@ +// Qwen3-TTS ECAPA-TDNN speaker encoder (ggml C++) — API publique. +// +// WAV mono 16-bit 24kHz -> mel 128 -> ECAPA-TDNN forward -> x_vector[1024] f32. +// Bit-correct vs torch.nn.functional.conv1d (cos > 0.999 vs Python ref). +// +// Architecture (cf speaker_encoder.gguf) : +// blocks.0 Conv1d(128 -> 512, k=5) +// blocks.1,2,3 SE-Res2Net (tdnn1 1x1 + 7x Conv k=3 + tdnn2 1x1 + se_block) +// mfa Conv1d(1536 -> 1536, k=1) // concat des 3 blocks +// asp.tdnn Conv1d(4608 -> 128, k=1) // Attentive Stats Pooling +// asp.conv Conv1d(128 -> 1536, k=1) +// fc Linear(3072 -> 1024) // x_vector final +// +// Constantes mel : SR=24000, N_FFT=1024, HOP=256, WIN=1024, N_MELS=128, log compression. +#pragma once +#include +#include + +struct SpeakerEncoder; // opaque + +// Charge GGUF + mel_basis (.bin float32 [128,513] = librosa pré-calculé). +// Retourne nullptr si échec. +SpeakerEncoder * speaker_encoder_load(const char * gguf_path, const char * mel_basis_path); + +// Encode un WAV mono 16-bit en x_vector[1024] f32. +// Le sample rate doit être 24000 (resample externe sinon). +// Renvoie un vecteur vide en cas d'échec. +std::vector speaker_encoder_encode_wav(SpeakerEncoder * spk, const char * wav_path); + +// Encode directement un buffer waveform mono 24kHz (f32 [-1..1]). +std::vector speaker_encoder_encode_waveform(SpeakerEncoder * spk, const std::vector & wav); + +void speaker_encoder_free(SpeakerEncoder * spk); diff --git a/dist/jni/tts_engine.cpp b/dist/jni/tts_engine.cpp index fc55f4c..a45b4ce 100644 --- a/dist/jni/tts_engine.cpp +++ b/dist/jni/tts_engine.cpp @@ -5,6 +5,7 @@ #include "sampler.h" #include "kazeia_text_tokenizer.h" #include "decoder.h" +#include "speaker_encoder.h" #include #include @@ -122,6 +123,9 @@ struct TtsEngine { KzTextTokenizer kz_tok; bool has_kz_tok = false; bool cp_use_cache = true; + + // --- Speaker encoder (optionnel, clonage vocal embarqué) --- + SpeakerEncoder * spk_enc = nullptr; // nullptr -> pas de clonage à la volée }; // =========================================================================== @@ -394,8 +398,17 @@ TtsEngine * tts_engine_load(const TtsEngineLoadCfg & cfg) { eng->codec_pad_emb.assign(eng->tok_embd.data() + (size_t)eng->codec_pad * hidden, eng->tok_embd.data() + (size_t)(eng->codec_pad + 1) * hidden); - fprintf(stderr, "tts_engine_load: %.2fs (talker_n_embd=%d, npe=%d, kz_tok=%d, cp_cache=%d)\n", - now_s() - t_load0, eng->n_embd, eng->npe, (int)eng->has_kz_tok, (int)cfg.cp_use_cache); + // --- Speaker encoder optionnel (clonage vocal embarqué) --- + if (cfg.speaker_encoder_gguf && cfg.mel_basis_path) { + eng->spk_enc = speaker_encoder_load(cfg.speaker_encoder_gguf, cfg.mel_basis_path); + if (!eng->spk_enc) { + fprintf(stderr, "tts_engine_load: speaker_encoder_load FAIL (continuing without cloning)\n"); + } + } + + fprintf(stderr, "tts_engine_load: %.2fs (talker_n_embd=%d, npe=%d, kz_tok=%d, cp_cache=%d, spk_enc=%d)\n", + now_s() - t_load0, eng->n_embd, eng->npe, (int)eng->has_kz_tok, + (int)cfg.cp_use_cache, (int)(eng->spk_enc != nullptr)); return eng; } @@ -415,6 +428,29 @@ TtsSynthesizeResult tts_engine_synthesize(TtsEngine * eng, const TtsSynthesizeCf const int n_embd = eng->n_embd; const int n_vocab = eng->n_vocab; + // --- 0) Override x_vector (clonage vocal) --- + // codec_input_emb a été pré-calculé au load() avec eng->xvector au slot 4. Si un + // override est fourni pour ce synth, on patche le slot puis on restaure via RAII + // (le synth peut return-early sur erreur, on évite de laisser l'engine sale). + const bool xvec_override = cfg.xvector_override && cfg.xvector_override_len == hidden; + if (cfg.xvector_override && !xvec_override) { + fprintf(stderr, "tts_engine_synthesize: xvector_override de longueur %d != hidden %d, ignoré\n", + cfg.xvector_override_len, hidden); + } + struct XvecRestore { + TtsEngine* e; int h; std::vector backup; + ~XvecRestore() { + if (!backup.empty()) + std::memcpy(e->codec_input_emb.data() + 4 * h, backup.data(), (size_t)h * sizeof(float)); + } + } _xvec_guard{eng, hidden, {}}; + if (xvec_override) { + _xvec_guard.backup.assign(eng->codec_input_emb.data() + 4 * hidden, + eng->codec_input_emb.data() + 5 * hidden); + std::memcpy(eng->codec_input_emb.data() + 4 * hidden, + cfg.xvector_override, (size_t)hidden * sizeof(float)); + } + // --- 1) Tokenize --- auto input_ids = kz_tok_encode_tts_prompt(eng->kz_tok, cfg.text); const int input_ids_len = (int)input_ids.size(); @@ -702,5 +738,24 @@ void tts_engine_free(TtsEngine * eng) { if (eng->talker_ctx) llama_free(eng->talker_ctx); if (eng->talker_m) llama_model_free(eng->talker_m); if (eng->has_kz_tok) kz_tok_free(eng->kz_tok); + if (eng->spk_enc) speaker_encoder_free(eng->spk_enc); delete eng; } + +// =========================================================================== +// SPEAKER ENCODER (clonage vocal embarqué) +// =========================================================================== + +std::vector tts_engine_encode_speaker_wav(TtsEngine * eng, const char * wav_path) { + if (!eng || !eng->spk_enc) { + fprintf(stderr, "tts_engine_encode_speaker_wav: speaker encoder non chargé\n"); + return {}; + } + return speaker_encoder_encode_wav(eng->spk_enc, wav_path); +} + +std::vector tts_engine_encode_speaker_waveform(TtsEngine * eng, const float * wav, int n_samples) { + if (!eng || !eng->spk_enc || !wav || n_samples <= 0) return {}; + std::vector w(wav, wav + n_samples); + return speaker_encoder_encode_waveform(eng->spk_enc, w); +} diff --git a/dist/jni/tts_engine.h b/dist/jni/tts_engine.h index 0337f09..1696321 100644 --- a/dist/jni/tts_engine.h +++ b/dist/jni/tts_engine.h @@ -10,6 +10,7 @@ #pragma once #include #include +#include struct TtsEngine; // opaque @@ -22,6 +23,13 @@ struct TtsEngineLoadCfg { bool use_htp = false; // true -> HTP0 prefill (option C), false -> CPU pur int n_threads = 6; bool cp_use_cache = true; // KV cache CP (cp_predict_cached) ; false -> cp_predict oracle + + // Clonage vocal embarqué (optionnel). Si renseignés, le speaker encoder est chargé en + // RAM (~17 MB f16) et permet de calculer un x_vector depuis n'importe quel WAV via + // tts_engine_encode_speaker_wav(). Sans ces deux fichiers, on garde le x_vector fixture + // (damien_xvector.bin) chargé une fois pour toutes. + const char * speaker_encoder_gguf = nullptr; // ex: speaker_encoder.gguf (17 MB f16) + const char * mel_basis_path = nullptr; // mel_basis_qwen3tts.bin (128*513 f32) }; struct TtsSynthesizeCfg { @@ -30,6 +38,12 @@ struct TtsSynthesizeCfg { int max_steps = 256; uint32_t seed = 42; + // Override du x_vector pour ce synth (clonage vocal). Si non-nullptr ET de longueur + // hidden (1024 en pratique), remplace le x_vector chargé. nullptr -> on garde le + // fixture chargé au load. Mémoire owned par l'appelant pour la durée du synth. + const float * xvector_override = nullptr; + int xvector_override_len = 0; + // Sampling : défauts validés à l'écoute (panel A/B) sur tts_engine + chraac subproc. // top_p=0.95 + rep_penalty=1.10 : sweet spot moins robotique, moins de pauses 690 // doublées sur CB0 (cf TU.2 attracteurs identifiés). Ancien défaut top_p=1.0 @@ -72,3 +86,11 @@ TtsSynthesizeResult tts_engine_synthesize(TtsEngine * eng, const TtsSynthesizeCf // Libère tout. void tts_engine_free(TtsEngine * eng); + +// Clonage vocal embarqué : encode un fichier WAV (mono 16-bit 24kHz) en x_vector[1024] f32. +// Requiert que tts_engine_load() ait reçu speaker_encoder_gguf + mel_basis_path. +// Renvoie vecteur vide si encoder non chargé ou échec. +std::vector tts_engine_encode_speaker_wav(TtsEngine * eng, const char * wav_path); + +// Idem mais depuis un buffer waveform en mémoire (mono 24kHz, f32 [-1..1]). +std::vector tts_engine_encode_speaker_waveform(TtsEngine * eng, const float * wav, int n_samples); diff --git a/dist/jni/tts_pipeline.cpp b/dist/jni/tts_pipeline.cpp index 8a7252d..5139307 100644 --- a/dist/jni/tts_pipeline.cpp +++ b/dist/jni/tts_pipeline.cpp @@ -56,10 +56,25 @@ int main(int argc, char** argv) { lc.use_htp = use_htp; lc.n_threads = env_i("KZTTS_THREADS", 6); lc.cp_use_cache = env_i("KZTTS_CP_CACHE", 1) != 0; + // Clonage vocal embarqué (opt-in via env) + lc.speaker_encoder_gguf = getenv("KZTTS_SPK_GGUF"); // ex: tts_dump/speaker_encoder.gguf + lc.mel_basis_path = getenv("KZTTS_MEL_BASIS"); // ex: tts_dump/mel_basis_qwen3tts.bin auto * eng = tts_engine_load(lc); if (!eng) { fprintf(stderr, "tts_engine_load FAIL\n"); return 3; } + // Si KZTTS_REF_WAV pointe sur un WAV mono 24kHz, on en extrait le x_vector et + // l'utilise comme override pour ce synth (clonage vocal in-process). + std::vector ref_xvec; + if (const char * ref_wav = getenv("KZTTS_REF_WAV")) { + ref_xvec = tts_engine_encode_speaker_wav(eng, ref_wav); + if (ref_xvec.empty()) { + fprintf(stderr, "tts_pipeline: KZTTS_REF_WAV=%s FAIL (encoder missing or read FAIL)\n", ref_wav); + } else { + fprintf(stderr, "tts_pipeline: KZTTS_REF_WAV=%s -> x_vector[%zu] OK\n", ref_wav, ref_xvec.size()); + } + } + TtsSynthesizeCfg sc; sc.text = kz_text; sc.out_wav_path = out_wav; @@ -73,6 +88,10 @@ int main(int argc, char** argv) { sc.talker_top_k = env_i("KZTTS_TOPK", 50); sc.talker_top_p = env_f("KZTTS_TOPP", 0.95f); sc.talker_rep_penalty = env_f("KZTTS_REPP", 1.10f); + if (!ref_xvec.empty()) { + sc.xvector_override = ref_xvec.data(); + sc.xvector_override_len = (int)ref_xvec.size(); + } auto R = tts_engine_synthesize(eng, sc); if (R.err) { fprintf(stderr, "tts_engine_synthesize FAIL err=%d\n", R.err); tts_engine_free(eng); return 4; }