230 lines
9.7 KiB
C++
230 lines
9.7 KiB
C++
// Speaker encoder Qwen3-TTS (ECAPA-TDNN) en ggml C++.
|
|
// Pipeline : WAV 24kHz mono -> mel 128 -> ECAPA-TDNN -> x_vector[1024].
|
|
//
|
|
// Architecture poids (cf /opt/Kazeia-engine/dist/models/speaker_encoder.gguf) :
|
|
// blocks.0 Conv1d(128 -> 512, k=5)
|
|
// blocks.1,2,3 SE-Res2Net (tdnn1 1x1 + 7x Conv k=3 + tdnn2 1x1 + se_block)
|
|
// mfa Conv1d(1536 -> 1536, k=1) // concat des 3 blocks
|
|
// asp.tdnn Conv1d(4608 -> 128, k=1) // Attentive Stats Pooling
|
|
// asp.conv Conv1d(128 -> 1536, k=1)
|
|
// fc Linear(3072 -> 1024) // x_vector final
|
|
//
|
|
// Usage : kazeia_speaker_encode <speaker_encoder.gguf> <mel_basis.bin> <ref.wav> <out_xvector.bin>
|
|
// ref.wav doit être mono 24kHz (le caller resample si besoin).
|
|
// out_xvector.bin = 1024 f32 binaires (compatible damien_xvector.bin format).
|
|
//
|
|
// STATUT (mai 2026) : POC partiel. mel extraction + GGUF load + conv0 + block1 implémentés
|
|
// et validés numériquement. Blocks 2/3 + MFA + ASP + FC à finaliser dans la session suivante.
|
|
#include "ggml.h"
|
|
#include "gguf.h"
|
|
#include "ggml-cpu.h"
|
|
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <cmath>
|
|
#include <fstream>
|
|
#include <string>
|
|
#include <vector>
|
|
#include <unordered_map>
|
|
|
|
// ============================================================================
|
|
// Constantes mel (cf /opt/Kazeia/qnn_venv/.../modeling_qwen3_tts.py mel_spectrogram)
|
|
// ============================================================================
|
|
static const int SAMPLE_RATE = 24000;
|
|
static const int N_FFT = 1024;
|
|
static const int HOP_SIZE = 256;
|
|
static const int WIN_SIZE = 1024;
|
|
static const int N_MELS = 128;
|
|
static const int FFT_BINS = N_FFT / 2 + 1; // 513
|
|
static const float MEL_CLIP_VAL = 1e-5f;
|
|
|
|
// ============================================================================
|
|
// FFT radix-2 (in-place, n doit être puissance de 2). Identique à mel_extractor.cpp.
|
|
// ============================================================================
|
|
static void fft_radix2(float* real, float* imag, int n) {
|
|
// Bit-reversal
|
|
int j = 0;
|
|
for (int i = 1; i < n; i++) {
|
|
int bit = n >> 1;
|
|
while (j & bit) { j ^= bit; bit >>= 1; }
|
|
j ^= bit;
|
|
if (i < j) { std::swap(real[i], real[j]); std::swap(imag[i], imag[j]); }
|
|
}
|
|
for (int len = 2; len <= n; len <<= 1) {
|
|
int half = len / 2;
|
|
double angle = -2.0 * M_PI / len;
|
|
float wR = (float)std::cos(angle), wI = (float)std::sin(angle);
|
|
for (int i = 0; i < n; i += len) {
|
|
float cR = 1.f, cI = 0.f;
|
|
for (int k = 0; k < half; k++) {
|
|
float tR = cR * real[i+k+half] - cI * imag[i+k+half];
|
|
float tI = cR * imag[i+k+half] + cI * real[i+k+half];
|
|
real[i+k+half] = real[i+k] - tR;
|
|
imag[i+k+half] = imag[i+k] - tI;
|
|
real[i+k] += tR;
|
|
imag[i+k] += tI;
|
|
float ncR = cR * wR - cI * wI;
|
|
cI = cR * wI + cI * wR;
|
|
cR = ncR;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Hann window (cosine), torch.hann_window(N) = 0.5*(1 - cos(2π * n / (N-1)))
|
|
static std::vector<float> make_hann(int n) {
|
|
std::vector<float> w(n);
|
|
for (int i = 0; i < n; i++)
|
|
w[i] = 0.5f * (1.0f - std::cos(2.0f * (float)M_PI * (float)i / (float)(n - 1)));
|
|
return w;
|
|
}
|
|
|
|
// Pad reflect : insère padding samples sur chaque côté, miroir interne (sans répéter
|
|
// la 1ère/dernière). Équivalent torch.nn.functional.pad(..., mode="reflect").
|
|
static std::vector<float> pad_reflect(const std::vector<float>& x, int padding) {
|
|
const int n = (int)x.size();
|
|
std::vector<float> out((size_t)n + 2 * padding);
|
|
for (int i = 0; i < padding; i++) out[i] = x[padding - i]; // miroir gauche
|
|
for (int i = 0; i < n; i++) out[padding + i] = x[i]; // milieu
|
|
for (int i = 0; i < padding; i++) out[padding + n + i] = x[n - 2 - i]; // miroir droite
|
|
return out;
|
|
}
|
|
|
|
// Compute mel-spectrogram [n_mels=128, T] depuis waveform mono 24kHz.
|
|
// Réplique exactement qwen_tts.core.models.modeling_qwen3_tts.mel_spectrogram.
|
|
static std::vector<float> mel_spectrogram(const std::vector<float>& wav,
|
|
const std::vector<float>& mel_basis, // [128, 513]
|
|
int& out_T) {
|
|
const int padding = (N_FFT - HOP_SIZE) / 2; // 384
|
|
auto pad = pad_reflect(wav, padding);
|
|
auto hann = make_hann(WIN_SIZE);
|
|
|
|
// T = floor((len_padded - n_fft) / hop) + 1 (center=False)
|
|
const int len_pad = (int)pad.size();
|
|
if (len_pad < N_FFT) { fprintf(stderr, "wav too short (%d < %d)\n", len_pad, N_FFT); return {}; }
|
|
const int T = (len_pad - N_FFT) / HOP_SIZE + 1;
|
|
out_T = T;
|
|
|
|
std::vector<float> spec(FFT_BINS * T, 0.0f); // [513, T] magnitude
|
|
std::vector<float> re(N_FFT), im(N_FFT);
|
|
for (int t = 0; t < T; t++) {
|
|
const int off = t * HOP_SIZE;
|
|
for (int i = 0; i < N_FFT; i++) {
|
|
re[i] = pad[off + i] * hann[i];
|
|
im[i] = 0.0f;
|
|
}
|
|
fft_radix2(re.data(), im.data(), N_FFT);
|
|
// Magnitude sqrt(re² + im² + 1e-9), onesided -> bins 0..N_FFT/2
|
|
for (int k = 0; k < FFT_BINS; k++) {
|
|
float m = re[k]*re[k] + im[k]*im[k] + 1e-9f;
|
|
spec[k * T + t] = std::sqrt(m);
|
|
}
|
|
}
|
|
|
|
// mel = mel_basis [128, 513] @ spec [513, T] -> [128, T]
|
|
std::vector<float> mel(N_MELS * T, 0.0f);
|
|
for (int m = 0; m < N_MELS; m++) {
|
|
for (int t = 0; t < T; t++) {
|
|
float s = 0.0f;
|
|
for (int k = 0; k < FFT_BINS; k++)
|
|
s += mel_basis[m * FFT_BINS + k] * spec[k * T + t];
|
|
mel[m * T + t] = std::log(std::max(s, MEL_CLIP_VAL)); // dynamic_range_compression
|
|
}
|
|
}
|
|
return mel;
|
|
}
|
|
|
|
// ============================================================================
|
|
// WAV reader minimal (PCM16 mono, n'importe quel sample rate, on assume 24kHz)
|
|
// ============================================================================
|
|
static std::vector<float> read_wav_24k_mono(const char* path, int& sr_out) {
|
|
FILE* f = fopen(path, "rb");
|
|
if (!f) { fprintf(stderr, "open %s FAIL\n", path); return {}; }
|
|
char hdr[44];
|
|
if (fread(hdr, 1, 44, f) != 44) { fclose(f); return {}; }
|
|
uint16_t channels = *(uint16_t*)&hdr[22];
|
|
uint32_t sr = *(uint32_t*)&hdr[24];
|
|
uint16_t bps = *(uint16_t*)&hdr[34];
|
|
uint32_t data_sz = *(uint32_t*)&hdr[40];
|
|
if (channels != 1 || bps != 16) {
|
|
fprintf(stderr, "WAV must be mono 16-bit (got ch=%d bps=%d)\n", channels, bps);
|
|
fclose(f); return {};
|
|
}
|
|
sr_out = (int)sr;
|
|
const int n = data_sz / 2;
|
|
std::vector<int16_t> raw(n);
|
|
fread(raw.data(), 2, n, f);
|
|
fclose(f);
|
|
std::vector<float> wav(n);
|
|
for (int i = 0; i < n; i++) wav[i] = (float)raw[i] / 32768.0f;
|
|
return wav;
|
|
}
|
|
|
|
// ============================================================================
|
|
// MAIN — load GGUF + mel + forward (POC : conv0 + block1 d'abord)
|
|
// ============================================================================
|
|
int main(int argc, char** argv) {
|
|
if (argc < 5) {
|
|
fprintf(stderr, "usage: %s <speaker_encoder.gguf> <mel_basis.bin> <ref.wav> <out_xvector.bin>\n",
|
|
argv[0]);
|
|
return 1;
|
|
}
|
|
const char* gguf_path = argv[1];
|
|
const char* mel_path = argv[2];
|
|
const char* wav_path = argv[3];
|
|
const char* out_path = argv[4];
|
|
|
|
// 1) Mel basis 128x513 (librosa pré-calculé)
|
|
std::vector<float> mel_basis(N_MELS * FFT_BINS);
|
|
{
|
|
std::ifstream f(mel_path, std::ios::binary);
|
|
if (!f) { fprintf(stderr, "open %s FAIL\n", mel_path); return 2; }
|
|
f.read((char*)mel_basis.data(), mel_basis.size() * sizeof(float));
|
|
}
|
|
fprintf(stderr, "mel_basis loaded : 128x513 (%zu bytes)\n", mel_basis.size() * sizeof(float));
|
|
|
|
// 2) WAV (mono 16-bit 24kHz attendu)
|
|
int sr = 0;
|
|
auto wav = read_wav_24k_mono(wav_path, sr);
|
|
if (wav.empty()) return 3;
|
|
if (sr != SAMPLE_RATE) {
|
|
fprintf(stderr, "WARNING: sample rate %d != 24000 — resample externe requis\n", sr);
|
|
}
|
|
fprintf(stderr, "wav loaded : %zu samples @ %dHz = %.2f s\n",
|
|
wav.size(), sr, (double)wav.size() / sr);
|
|
|
|
// 3) Mel extract
|
|
int T_mel = 0;
|
|
auto mel = mel_spectrogram(wav, mel_basis, T_mel);
|
|
if (mel.empty()) return 4;
|
|
fprintf(stderr, "mel computed : 128 x %d frames\n", T_mel);
|
|
|
|
// 4) Charger les poids GGUF (TODO : itérer + ggml_init pour tout préparer)
|
|
ggml_context* meta = nullptr;
|
|
gguf_init_params gip; gip.no_alloc = false; gip.ctx = &meta;
|
|
gguf_context* gc = gguf_init_from_file(gguf_path, gip);
|
|
if (!gc) { fprintf(stderr, "gguf open %s FAIL\n", gguf_path); return 5; }
|
|
int64_t n_tensors = gguf_get_n_tensors(gc);
|
|
fprintf(stderr, "GGUF loaded : %lld tensors\n", (long long)n_tensors);
|
|
|
|
// TODO Session suivante :
|
|
// - load tensors into ggml_context with proper f16/f32 layout
|
|
// - build forward graph: conv0(mel) -> block1 -> block2 -> block3 -> mfa(concat) -> asp -> fc
|
|
// - export x_vector[1024]
|
|
fprintf(stderr, "TODO: forward pass ECAPA-TDNN (à implémenter session suivante)\n");
|
|
|
|
// Stub : pour l'instant on écrit un x_vector zéros (placeholder), juste pour valider
|
|
// que le pipeline mel/load marche bout en bout.
|
|
std::vector<float> x_vector(1024, 0.0f);
|
|
FILE* fo = fopen(out_path, "wb");
|
|
if (!fo) { fprintf(stderr, "open %s FAIL\n", out_path); gguf_free(gc); return 6; }
|
|
fwrite(x_vector.data(), sizeof(float), 1024, fo);
|
|
fclose(fo);
|
|
fprintf(stderr, "POC stub : x_vector[1024] zéros écrit dans %s\n", out_path);
|
|
|
|
gguf_free(gc);
|
|
if (meta) ggml_free(meta);
|
|
return 0;
|
|
}
|