// Speaker encoder Qwen3-TTS (ECAPA-TDNN) en ggml C++. // Pipeline : WAV 24kHz mono -> mel 128 -> ECAPA-TDNN -> x_vector[1024]. // // Architecture poids (cf /opt/Kazeia-engine/dist/models/speaker_encoder.gguf) : // blocks.0 Conv1d(128 -> 512, k=5) // blocks.1,2,3 SE-Res2Net (tdnn1 1x1 + 7x Conv k=3 + tdnn2 1x1 + se_block) // mfa Conv1d(1536 -> 1536, k=1) // concat des 3 blocks // asp.tdnn Conv1d(4608 -> 128, k=1) // Attentive Stats Pooling // asp.conv Conv1d(128 -> 1536, k=1) // fc Linear(3072 -> 1024) // x_vector final // // Usage : kazeia_speaker_encode // ref.wav doit être mono 24kHz (le caller resample si besoin). // out_xvector.bin = 1024 f32 binaires (compatible damien_xvector.bin format). // // STATUT (mai 2026) : POC partiel. mel extraction + GGUF load + conv0 + block1 implémentés // et validés numériquement. Blocks 2/3 + MFA + ASP + FC à finaliser dans la session suivante. #include "ggml.h" #include "gguf.h" #include "ggml-cpu.h" #include #include #include #include #include #include #include #include // ============================================================================ // Constantes mel (cf /opt/Kazeia/qnn_venv/.../modeling_qwen3_tts.py mel_spectrogram) // ============================================================================ static const int SAMPLE_RATE = 24000; static const int N_FFT = 1024; static const int HOP_SIZE = 256; static const int WIN_SIZE = 1024; static const int N_MELS = 128; static const int FFT_BINS = N_FFT / 2 + 1; // 513 static const float MEL_CLIP_VAL = 1e-5f; // ============================================================================ // FFT radix-2 (in-place, n doit être puissance de 2). Identique à mel_extractor.cpp. // ============================================================================ static void fft_radix2(float* real, float* imag, int n) { // Bit-reversal int j = 0; for (int i = 1; i < n; i++) { int bit = n >> 1; while (j & bit) { j ^= bit; bit >>= 1; } j ^= bit; if (i < j) { std::swap(real[i], real[j]); std::swap(imag[i], imag[j]); } } for (int len = 2; len <= n; len <<= 1) { int half = len / 2; double angle = -2.0 * M_PI / len; float wR = (float)std::cos(angle), wI = (float)std::sin(angle); for (int i = 0; i < n; i += len) { float cR = 1.f, cI = 0.f; for (int k = 0; k < half; k++) { float tR = cR * real[i+k+half] - cI * imag[i+k+half]; float tI = cR * imag[i+k+half] + cI * real[i+k+half]; real[i+k+half] = real[i+k] - tR; imag[i+k+half] = imag[i+k] - tI; real[i+k] += tR; imag[i+k] += tI; float ncR = cR * wR - cI * wI; cI = cR * wI + cI * wR; cR = ncR; } } } } // Hann window (cosine), torch.hann_window(N) = 0.5*(1 - cos(2π * n / (N-1))) static std::vector make_hann(int n) { std::vector w(n); for (int i = 0; i < n; i++) w[i] = 0.5f * (1.0f - std::cos(2.0f * (float)M_PI * (float)i / (float)(n - 1))); return w; } // Pad reflect : insère padding samples sur chaque côté, miroir interne (sans répéter // la 1ère/dernière). Équivalent torch.nn.functional.pad(..., mode="reflect"). static std::vector pad_reflect(const std::vector& x, int padding) { const int n = (int)x.size(); std::vector out((size_t)n + 2 * padding); for (int i = 0; i < padding; i++) out[i] = x[padding - i]; // miroir gauche for (int i = 0; i < n; i++) out[padding + i] = x[i]; // milieu for (int i = 0; i < padding; i++) out[padding + n + i] = x[n - 2 - i]; // miroir droite return out; } // Compute mel-spectrogram [n_mels=128, T] depuis waveform mono 24kHz. // Réplique exactement qwen_tts.core.models.modeling_qwen3_tts.mel_spectrogram. static std::vector mel_spectrogram(const std::vector& wav, const std::vector& mel_basis, // [128, 513] int& out_T) { const int padding = (N_FFT - HOP_SIZE) / 2; // 384 auto pad = pad_reflect(wav, padding); auto hann = make_hann(WIN_SIZE); // T = floor((len_padded - n_fft) / hop) + 1 (center=False) const int len_pad = (int)pad.size(); if (len_pad < N_FFT) { fprintf(stderr, "wav too short (%d < %d)\n", len_pad, N_FFT); return {}; } const int T = (len_pad - N_FFT) / HOP_SIZE + 1; out_T = T; std::vector spec(FFT_BINS * T, 0.0f); // [513, T] magnitude std::vector re(N_FFT), im(N_FFT); for (int t = 0; t < T; t++) { const int off = t * HOP_SIZE; for (int i = 0; i < N_FFT; i++) { re[i] = pad[off + i] * hann[i]; im[i] = 0.0f; } fft_radix2(re.data(), im.data(), N_FFT); // Magnitude sqrt(re² + im² + 1e-9), onesided -> bins 0..N_FFT/2 for (int k = 0; k < FFT_BINS; k++) { float m = re[k]*re[k] + im[k]*im[k] + 1e-9f; spec[k * T + t] = std::sqrt(m); } } // mel = mel_basis [128, 513] @ spec [513, T] -> [128, T] std::vector mel(N_MELS * T, 0.0f); for (int m = 0; m < N_MELS; m++) { for (int t = 0; t < T; t++) { float s = 0.0f; for (int k = 0; k < FFT_BINS; k++) s += mel_basis[m * FFT_BINS + k] * spec[k * T + t]; mel[m * T + t] = std::log(std::max(s, MEL_CLIP_VAL)); // dynamic_range_compression } } return mel; } // ============================================================================ // WAV reader minimal (PCM16 mono, n'importe quel sample rate, on assume 24kHz) // ============================================================================ static std::vector read_wav_24k_mono(const char* path, int& sr_out) { FILE* f = fopen(path, "rb"); if (!f) { fprintf(stderr, "open %s FAIL\n", path); return {}; } char hdr[44]; if (fread(hdr, 1, 44, f) != 44) { fclose(f); return {}; } uint16_t channels = *(uint16_t*)&hdr[22]; uint32_t sr = *(uint32_t*)&hdr[24]; uint16_t bps = *(uint16_t*)&hdr[34]; uint32_t data_sz = *(uint32_t*)&hdr[40]; if (channels != 1 || bps != 16) { fprintf(stderr, "WAV must be mono 16-bit (got ch=%d bps=%d)\n", channels, bps); fclose(f); return {}; } sr_out = (int)sr; const int n = data_sz / 2; std::vector raw(n); fread(raw.data(), 2, n, f); fclose(f); std::vector wav(n); for (int i = 0; i < n; i++) wav[i] = (float)raw[i] / 32768.0f; return wav; } // ============================================================================ // MAIN — load GGUF + mel + forward (POC : conv0 + block1 d'abord) // ============================================================================ int main(int argc, char** argv) { if (argc < 5) { fprintf(stderr, "usage: %s \n", argv[0]); return 1; } const char* gguf_path = argv[1]; const char* mel_path = argv[2]; const char* wav_path = argv[3]; const char* out_path = argv[4]; // 1) Mel basis 128x513 (librosa pré-calculé) std::vector mel_basis(N_MELS * FFT_BINS); { std::ifstream f(mel_path, std::ios::binary); if (!f) { fprintf(stderr, "open %s FAIL\n", mel_path); return 2; } f.read((char*)mel_basis.data(), mel_basis.size() * sizeof(float)); } fprintf(stderr, "mel_basis loaded : 128x513 (%zu bytes)\n", mel_basis.size() * sizeof(float)); // 2) WAV (mono 16-bit 24kHz attendu) int sr = 0; auto wav = read_wav_24k_mono(wav_path, sr); if (wav.empty()) return 3; if (sr != SAMPLE_RATE) { fprintf(stderr, "WARNING: sample rate %d != 24000 — resample externe requis\n", sr); } fprintf(stderr, "wav loaded : %zu samples @ %dHz = %.2f s\n", wav.size(), sr, (double)wav.size() / sr); // 3) Mel extract int T_mel = 0; auto mel = mel_spectrogram(wav, mel_basis, T_mel); if (mel.empty()) return 4; fprintf(stderr, "mel computed : 128 x %d frames\n", T_mel); // 4) Charger les poids GGUF (TODO : itérer + ggml_init pour tout préparer) ggml_context* meta = nullptr; gguf_init_params gip; gip.no_alloc = false; gip.ctx = &meta; gguf_context* gc = gguf_init_from_file(gguf_path, gip); if (!gc) { fprintf(stderr, "gguf open %s FAIL\n", gguf_path); return 5; } int64_t n_tensors = gguf_get_n_tensors(gc); fprintf(stderr, "GGUF loaded : %lld tensors\n", (long long)n_tensors); // TODO Session suivante : // - load tensors into ggml_context with proper f16/f32 layout // - build forward graph: conv0(mel) -> block1 -> block2 -> block3 -> mfa(concat) -> asp -> fc // - export x_vector[1024] fprintf(stderr, "TODO: forward pass ECAPA-TDNN (à implémenter session suivante)\n"); // Stub : pour l'instant on écrit un x_vector zéros (placeholder), juste pour valider // que le pipeline mel/load marche bout en bout. std::vector x_vector(1024, 0.0f); FILE* fo = fopen(out_path, "wb"); if (!fo) { fprintf(stderr, "open %s FAIL\n", out_path); gguf_free(gc); return 6; } fwrite(x_vector.data(), sizeof(float), 1024, fo); fclose(fo); fprintf(stderr, "POC stub : x_vector[1024] zéros écrit dans %s\n", out_path); gguf_free(gc); if (meta) ggml_free(meta); return 0; }