444 lines
23 KiB
C++
444 lines
23 KiB
C++
// Pipeline TTS bout-en-bout EN UN SEUL BINAIRE sur tablette :
|
|
// texte (input_ids déjà tokenisés) + x_vector
|
|
// -> construction prefill_embeds (text_projection + tok_embd + spéciaux)
|
|
// -> talker engine (libllama, M-RoPE, embeds-mode, Patch 1+2)
|
|
// -> sampling greedy CB0 + CP (cp_inference, recompute bit-exact)
|
|
// -> codes [T, 16]
|
|
// -> decoder ggml (libqwen3tts-decoder rebuilt vs ql/ggml)
|
|
// -> PCM 24 kHz WAV
|
|
//
|
|
// Aucune dépendance Python à l'exécution. Tokenizer text + speaker encoder restent
|
|
// offline (input_ids pré-calculé pour la phrase, x_vector pré-calculé pour la voix).
|
|
//
|
|
// Usage: tts_pipeline <talker_gguf> <dump_dir> <out.wav> [cpu|htp] [max_steps]
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <cmath>
|
|
#include <cstdint>
|
|
#include <vector>
|
|
#include <string>
|
|
#include <fstream>
|
|
#include <chrono>
|
|
#include "llama.h"
|
|
#include "ggml-backend.h"
|
|
#include "cp_inference.h"
|
|
#include "sampler.h"
|
|
#include "kazeia_text_tokenizer.h"
|
|
#include "decoder.h" // Kazeia decoder ggml (linké via libqwen3tts-decoder.a)
|
|
|
|
using namespace kazeia::tts;
|
|
|
|
static double now_s() {
|
|
using clk = std::chrono::steady_clock;
|
|
return std::chrono::duration<double>(clk::now().time_since_epoch()).count();
|
|
}
|
|
|
|
static std::vector<float> read_f32(const std::string& p, size_t n_expected) {
|
|
std::ifstream f(p, std::ios::binary | std::ios::ate);
|
|
if (!f) { fprintf(stderr, "open %s\n", p.c_str()); exit(1); }
|
|
size_t n = (size_t)f.tellg() / sizeof(float);
|
|
if (n_expected && n != n_expected) { fprintf(stderr, "%s: %zu f32, attendu %zu\n", p.c_str(), n, n_expected); exit(1); }
|
|
f.seekg(0); std::vector<float> v(n); f.read((char*)v.data(), n * sizeof(float)); return v;
|
|
}
|
|
static std::vector<int32_t> read_i32(const std::string& p, size_t n_expected) {
|
|
std::ifstream f(p, std::ios::binary | std::ios::ate);
|
|
if (!f) { fprintf(stderr, "open %s\n", p.c_str()); exit(1); }
|
|
size_t n = (size_t)f.tellg() / sizeof(int32_t);
|
|
if (n_expected && n != n_expected) { fprintf(stderr, "%s: %zu i32, attendu %zu\n", p.c_str(), n, n_expected); exit(1); }
|
|
f.seekg(0); std::vector<int32_t> v(n); f.read((char*)v.data(), n * sizeof(int32_t)); return v;
|
|
}
|
|
static ggml_backend_dev_t find_htp() {
|
|
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
|
auto d = ggml_backend_dev_get(i);
|
|
if (!strcmp(ggml_backend_dev_name(d), "HTP0")) return d;
|
|
}
|
|
return nullptr;
|
|
}
|
|
static inline float silu(float x) { return x / (1.0f + std::exp(-x)); }
|
|
static void linear(const float* W, const float* b, int out_dim, int in_dim, const float* x, float* out) {
|
|
for (int m = 0; m < out_dim; ++m) {
|
|
float s = b ? b[m] : 0.0f;
|
|
const float* wr = W + (size_t)m * in_dim;
|
|
for (int k = 0; k < in_dim; ++k) s += wr[k] * x[k];
|
|
out[m] = s;
|
|
}
|
|
}
|
|
static void text_projection(const float* in, int N,
|
|
const float* fc1_w, const float* fc1_b,
|
|
const float* fc2_w, const float* fc2_b,
|
|
float* mid_buf, float* out) {
|
|
for (int n = 0; n < N; ++n) {
|
|
linear(fc1_w, fc1_b, 2048, 2048, in + n * 2048, mid_buf);
|
|
for (int d = 0; d < 2048; ++d) mid_buf[d] = silu(mid_buf[d]);
|
|
linear(fc2_w, fc2_b, 1024, 2048, mid_buf, out + n * 1024);
|
|
}
|
|
}
|
|
|
|
static bool write_wav_pcm16_mono(const char* path, const float* samples, size_t N, int sr = 24000) {
|
|
FILE* f = fopen(path, "wb");
|
|
if (!f) return false;
|
|
const uint32_t data_bytes = (uint32_t)(N * 2);
|
|
const uint32_t riff_size = 36 + data_bytes;
|
|
auto w16 = [&](uint16_t v){ fwrite(&v, 2, 1, f); };
|
|
auto w32 = [&](uint32_t v){ fwrite(&v, 4, 1, f); };
|
|
fwrite("RIFF", 1, 4, f); w32(riff_size); fwrite("WAVE", 1, 4, f);
|
|
fwrite("fmt ", 1, 4, f); w32(16); w16(1); w16(1); // pcm mono
|
|
w32((uint32_t)sr); w32((uint32_t)sr * 2); w16(2); w16(16);
|
|
fwrite("data", 1, 4, f); w32(data_bytes);
|
|
for (size_t i = 0; i < N; ++i) {
|
|
float v = samples[i]; if (v > 1.f) v = 1.f; else if (v < -1.f) v = -1.f;
|
|
int16_t pcm = (int16_t)std::lround(v * 32767.0f);
|
|
fwrite(&pcm, 2, 1, f);
|
|
}
|
|
fclose(f); return true;
|
|
}
|
|
|
|
int main(int argc, char** argv) {
|
|
if (argc < 4) {
|
|
printf("usage: %s <talker_gguf> <dump_dir> <out.wav> [cpu|htp] [max_steps]\n", argv[0]);
|
|
printf(" Texte arbitraire (au lieu de input_ids_full.bin dumpé) :\n");
|
|
printf(" KZTTS_VOCAB_GGUF=/path/to/qwen3.gguf KZTTS_TEXT=\"phrase libre\" %s ...\n", argv[0]);
|
|
return 1;
|
|
}
|
|
const char* gguf = argv[1];
|
|
std::string D = argv[2]; if (D.back() != '/') D += '/';
|
|
const char* OUT_WAV = argv[3];
|
|
bool force_cpu = (argc >= 5 && !strcmp(argv[4], "cpu"));
|
|
int max_steps_arg = (argc >= 6) ? atoi(argv[5]) : 64;
|
|
// Texte arbitraire : si KZTTS_TEXT et KZTTS_VOCAB_GGUF sont posés, on tokenize la phrase
|
|
// au lieu de relire input_ids_full.bin. Vérifié bit-exact contre le golden HF tokenizer
|
|
// sur "Bonjour je m'appelle Kazeia" via test_tokenizer.
|
|
const char * kz_text = getenv("KZTTS_TEXT");
|
|
const char * kz_vocab_gguf = getenv("KZTTS_VOCAB_GGUF");
|
|
const bool use_kz_tok = (kz_text && kz_vocab_gguf && *kz_text && *kz_vocab_gguf);
|
|
|
|
// ----- 0) Constants from manifest_text
|
|
int text_vocab = 151936, text_hidden = 2048, hidden = 1024;
|
|
int tts_bos = 151672, tts_eos = 151673, tts_pad = 151671;
|
|
int codec_bos = 2149, codec_eos = 2150, codec_pad = 2148;
|
|
int codec_think = 2154, codec_nothink = 2155, codec_think_bos = 2156, codec_think_eos = 2157;
|
|
int lang_fr = 2061;
|
|
int input_ids_len = 16;
|
|
{
|
|
std::ifstream f(D + "manifest_text.txt"); if (!f) { fprintf(stderr, "no manifest_text\n"); return 1; }
|
|
std::string line;
|
|
auto eq = [&](const char* k){ return line.rfind(k, 0) == 0; };
|
|
auto val = [&](size_t off){ return atoi(line.c_str() + off); };
|
|
while (std::getline(f, line)) {
|
|
if (eq("text_vocab_size:")) text_vocab = val(16);
|
|
else if (eq("text_hidden_size:")) text_hidden = val(17);
|
|
else if (eq("hidden_size:")) hidden = val(12);
|
|
else if (eq("tts_bos_token_id:")) tts_bos = val(17);
|
|
else if (eq("tts_eos_token_id:")) tts_eos = val(17);
|
|
else if (eq("tts_pad_token_id:")) tts_pad = val(17);
|
|
else if (eq("codec_bos_id:")) codec_bos = val(13);
|
|
else if (eq("codec_eos_id:")) codec_eos = val(13);
|
|
else if (eq("codec_pad_id:")) codec_pad = val(13);
|
|
else if (eq("codec_think_id:")) codec_think = val(15);
|
|
else if (eq("codec_nothink_id:")) codec_nothink = val(17);
|
|
else if (eq("codec_think_bos_id:")) codec_think_bos = val(19);
|
|
else if (eq("codec_think_eos_id:")) codec_think_eos = val(19);
|
|
else if (eq("codec_language_french:")) lang_fr = val(22);
|
|
else if (eq("input_ids_len:")) input_ids_len = val(14);
|
|
}
|
|
}
|
|
(void)codec_nothink; (void)tts_eos;
|
|
printf("manifest: input_ids_len=%d hid=%d text_hid=%d\n", input_ids_len, hidden, text_hidden);
|
|
|
|
// ----- 1) Charger toutes les fixtures
|
|
const double t_load0 = now_s();
|
|
auto text_embed = read_f32(D + "text_embed.bin", (size_t)text_vocab * text_hidden);
|
|
auto tp_fc1_w = read_f32(D + "tp_fc1_w.bin", (size_t)text_hidden * text_hidden);
|
|
auto tp_fc1_b = read_f32(D + "tp_fc1_b.bin", (size_t)text_hidden);
|
|
auto tp_fc2_w = read_f32(D + "tp_fc2_w.bin", (size_t)hidden * text_hidden);
|
|
auto tp_fc2_b = read_f32(D + "tp_fc2_b.bin", (size_t)hidden);
|
|
auto tok_embd = read_f32(D + "talker_tok_embd.bin", (size_t)3072 * hidden);
|
|
auto xvector = read_f32(D + "damien_xvector.bin", (size_t)hidden);
|
|
|
|
// input_ids : depuis le tokenizer C++ (texte arbitraire) ou depuis le dump golden.
|
|
// L'engin de l'app initialisera llama_backend_init plus bas dans la section talker ;
|
|
// pour pouvoir charger le vocab maintenant, on l'initialise dès ici (idempotent côté llama).
|
|
std::vector<int32_t> input_ids;
|
|
KzTextTokenizer kz_tok;
|
|
if (use_kz_tok) {
|
|
// Init backend AVANT chargement vocab. Le talker plus bas ré-init (idempotent côté
|
|
// llama). HMX off posé ici par sécurité (le talker le repose ensuite ; sans ça un
|
|
// init précoce du backend pourrait verrouiller HMX=on selon l'archi du vocab gguf).
|
|
setenv("GGML_HEXAGON_USE_HMX", "0", 1);
|
|
llama_backend_init();
|
|
if (!kz_tok_load(kz_tok, kz_vocab_gguf)) { printf("vocab load FAILED: %s\n", kz_vocab_gguf); return 1; }
|
|
input_ids = kz_tok_encode_tts_prompt(kz_tok, kz_text);
|
|
input_ids_len = (int)input_ids.size();
|
|
if (input_ids_len < 8) { printf("kz_tok: trop court (%d, min=8)\n", input_ids_len); return 1; }
|
|
printf("kz_tok: \"%s\" -> %d tokens\n", kz_text, input_ids_len);
|
|
} else {
|
|
auto v = read_i32(D + "input_ids_full.bin", (size_t)input_ids_len);
|
|
input_ids = std::move(v);
|
|
}
|
|
auto tts_pad_emb_ref = read_f32(D + "tts_pad_embed.bin", (size_t)hidden); // sanity
|
|
(void)tts_pad_emb_ref;
|
|
|
|
// ----- 2) Construire prefill_embeds (cf. build_prefill.cpp, validé bit-exact)
|
|
const double t_pf0 = now_s();
|
|
std::vector<float> mid(text_hidden);
|
|
// special tokens
|
|
std::vector<float> spec_text(3 * text_hidden);
|
|
int sp[3] = { tts_bos, tts_eos, tts_pad };
|
|
for (int i = 0; i < 3; ++i)
|
|
std::memcpy(spec_text.data() + i * text_hidden, text_embed.data() + (size_t)sp[i] * text_hidden,
|
|
text_hidden * sizeof(float));
|
|
std::vector<float> spec_proj(3 * hidden);
|
|
text_projection(spec_text.data(), 3, tp_fc1_w.data(), tp_fc1_b.data(),
|
|
tp_fc2_w.data(), tp_fc2_b.data(), mid.data(), spec_proj.data());
|
|
const float* tts_bos_emb = spec_proj.data() + 0 * hidden;
|
|
const float* tts_eos_emb = spec_proj.data() + 1 * hidden;
|
|
const float* tts_pad_emb = spec_proj.data() + 2 * hidden;
|
|
|
|
// codec prefix [think, think_bos, lang_fr, think_eos, x_vector, codec_pad, codec_bos]
|
|
int codec_prefill[4] = { codec_think, codec_think_bos, lang_fr, codec_think_eos };
|
|
std::vector<float> codec_input_emb(7 * hidden);
|
|
for (int i = 0; i < 4; ++i)
|
|
std::memcpy(codec_input_emb.data() + i * hidden, tok_embd.data() + (size_t)codec_prefill[i] * hidden,
|
|
hidden * sizeof(float));
|
|
std::memcpy(codec_input_emb.data() + 4 * hidden, xvector.data(), hidden * sizeof(float));
|
|
std::memcpy(codec_input_emb.data() + 5 * hidden, tok_embd.data() + (size_t)codec_pad * hidden, hidden * sizeof(float));
|
|
std::memcpy(codec_input_emb.data() + 6 * hidden, tok_embd.data() + (size_t)codec_bos * hidden, hidden * sizeof(float));
|
|
|
|
// role
|
|
std::vector<float> role_text(3 * text_hidden);
|
|
for (int i = 0; i < 3; ++i)
|
|
std::memcpy(role_text.data() + i * text_hidden, text_embed.data() + (size_t)input_ids[i] * text_hidden,
|
|
text_hidden * sizeof(float));
|
|
std::vector<float> role_proj(3 * hidden);
|
|
text_projection(role_text.data(), 3, tp_fc1_w.data(), tp_fc1_b.data(),
|
|
tp_fc2_w.data(), tp_fc2_b.data(), mid.data(), role_proj.data());
|
|
|
|
// text body
|
|
const int Nt = input_ids_len - 5 - 3;
|
|
std::vector<float> body_text((size_t)Nt * text_hidden);
|
|
for (int i = 0; i < Nt; ++i)
|
|
std::memcpy(body_text.data() + i * text_hidden, text_embed.data() + (size_t)input_ids[3 + i] * text_hidden,
|
|
text_hidden * sizeof(float));
|
|
std::vector<float> body_proj((size_t)Nt * hidden);
|
|
text_projection(body_text.data(), Nt, tp_fc1_w.data(), tp_fc1_b.data(),
|
|
tp_fc2_w.data(), tp_fc2_b.data(), mid.data(), body_proj.data());
|
|
|
|
// assemble
|
|
const int T_prefill = 3 + 6 + (Nt + 1) + 1;
|
|
std::vector<float> prefill((size_t)T_prefill * hidden, 0.0f);
|
|
std::memcpy(prefill.data(), role_proj.data(), 3 * hidden * sizeof(float));
|
|
for (int i = 0; i < 5; ++i) {
|
|
const float* ce = codec_input_emb.data() + i * hidden;
|
|
float* p = prefill.data() + (3 + i) * hidden;
|
|
for (int d = 0; d < hidden; ++d) p[d] = tts_pad_emb[d] + ce[d];
|
|
}
|
|
{ const float* ce = codec_input_emb.data() + 5 * hidden;
|
|
float* p = prefill.data() + 8 * hidden;
|
|
for (int d = 0; d < hidden; ++d) p[d] = tts_bos_emb[d] + ce[d]; }
|
|
const float* codec_pad_emb = tok_embd.data() + (size_t)codec_pad * hidden;
|
|
for (int i = 0; i < Nt; ++i) {
|
|
const float* bp = body_proj.data() + i * hidden;
|
|
float* p = prefill.data() + (9 + i) * hidden;
|
|
for (int d = 0; d < hidden; ++d) p[d] = bp[d] + codec_pad_emb[d];
|
|
}
|
|
{ float* p = prefill.data() + (9 + Nt) * hidden;
|
|
for (int d = 0; d < hidden; ++d) p[d] = tts_eos_emb[d] + codec_pad_emb[d]; }
|
|
{ const float* cbe = tok_embd.data() + (size_t)codec_bos * hidden;
|
|
float* p = prefill.data() + (10 + Nt) * hidden;
|
|
for (int d = 0; d < hidden; ++d) p[d] = tts_pad_emb[d] + cbe[d]; }
|
|
printf("prefill constructed T=%d en %.3f s\n", T_prefill, now_s() - t_pf0);
|
|
|
|
// ----- 3) Charger talker (engine, libllama)
|
|
setenv("GGML_HEXAGON_USE_HMX", "0", 1);
|
|
llama_backend_init();
|
|
auto mp = llama_model_default_params();
|
|
ggml_backend_dev_t devs[2] = { force_cpu ? nullptr : find_htp(), nullptr };
|
|
if (devs[0]) { mp.n_gpu_layers = 99; mp.devices = devs; printf("talker: HTP0\n"); }
|
|
else { mp.n_gpu_layers = 0; printf("talker: CPU\n"); }
|
|
auto m = llama_model_load_from_file(gguf, mp);
|
|
if (!m) { printf("talker load FAILED\n"); return 1; }
|
|
auto cp = llama_context_default_params();
|
|
cp.n_ctx = std::max(512, T_prefill + max_steps_arg + 16); cp.n_batch = 1024;
|
|
cp.n_threads = (getenv("KZTTS_THREADS") ? atoi(getenv("KZTTS_THREADS")) : 6);
|
|
cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
|
|
cp.embeddings = true;
|
|
auto ctx = llama_init_from_model(m, cp);
|
|
if (!ctx) { printf("talker ctx FAILED\n"); return 1; }
|
|
const int n_embd = llama_model_n_embd(m);
|
|
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(m));
|
|
if (n_embd != hidden || n_vocab != 3072) { printf("talker dims mismatch (n_embd=%d, n_vocab=%d)\n", n_embd, n_vocab); return 1; }
|
|
printf("talker: n_embd=%d n_vocab=%d threads=%d\n", n_embd, n_vocab, cp.n_threads);
|
|
|
|
// ----- 4) Charger CP + activer sampling HF-style (rep_penalty + top_k + top_p + temp).
|
|
// Sans sampling propre, le greedy tombe dans des attracteurs (codes 690/207/1571 répétés
|
|
// = silence/pause) -> WAV avec gros blancs. rep_penalty=1.05 = Python subtalker défaut.
|
|
// KZTTS_CP_CACHE=1 -> cp_predict_cached (1 prefill + 14 decode via KV cache, ~8x speedup).
|
|
// KZTTS_CP_CACHE=0 (défaut) -> cp_predict (oracle bit-exact, 15 recomputes complets).
|
|
const bool cp_use_cache = (getenv("KZTTS_CP_CACHE") && atoi(getenv("KZTTS_CP_CACHE")) != 0);
|
|
printf("CP path: %s\n", cp_use_cache ? "CACHED (KZTTS_CP_CACHE=1)" : "RECOMPUTE (oracle bit-exact)");
|
|
CPState cp_state; cp_state.n_threads = cp.n_threads;
|
|
cp_state.sampler.temp = getenv("KZTTS_CP_TEMP") ? atof(getenv("KZTTS_CP_TEMP")) : 0.9f;
|
|
cp_state.sampler.top_k = getenv("KZTTS_CP_TOPK") ? atoi(getenv("KZTTS_CP_TOPK")) : 50;
|
|
cp_state.sampler.top_p = getenv("KZTTS_CP_TOPP") ? atof(getenv("KZTTS_CP_TOPP")) : 1.0f;
|
|
cp_state.sampler.rep_penalty = getenv("KZTTS_CP_REPP") ? atof(getenv("KZTTS_CP_REPP")) : 1.05f;
|
|
cp_state.sampler.rep_window = 16;
|
|
if (!cp_load(cp_state, (D + "cp_f16.gguf").c_str(), (D + "cp_heads.bin").c_str(), (D + "cp_codec_embs.bin").c_str())) {
|
|
printf("CP load FAILED\n"); return 1;
|
|
}
|
|
const uint32_t SEED = getenv("KZTTS_SEED") ? atoi(getenv("KZTTS_SEED")) : 42;
|
|
sampler_seed(cp_state.sampler, SEED + 1); // seed différent du Talker pour décorréler
|
|
printf("CP sampling: temp=%.2f top_k=%d top_p=%.2f rep_penalty=%.2f\n",
|
|
cp_state.sampler.temp, cp_state.sampler.top_k, cp_state.sampler.top_p, cp_state.sampler.rep_penalty);
|
|
|
|
// ----- 5) Charger decoder
|
|
Decoder dec;
|
|
if (!dec.load((D + "qwen3tts_decoder.gguf").c_str())) { printf("decoder load FAILED\n"); return 1; }
|
|
|
|
printf("=== load total : %.3f s ===\n", now_s() - t_load0);
|
|
|
|
// ----- 6) Pipeline
|
|
auto rt = llama_model_rope_type(m);
|
|
const int npe = (rt == LLAMA_ROPE_TYPE_MROPE || rt == LLAMA_ROPE_TYPE_IMROPE) ? 4 : 1;
|
|
|
|
// --- PREFILL talker
|
|
const double t_pfill0 = now_s();
|
|
{
|
|
std::vector<llama_pos> pos(T_prefill * npe, 0);
|
|
std::vector<int32_t> nsd(T_prefill, 1);
|
|
std::vector<llama_seq_id> sid0(T_prefill, 0);
|
|
std::vector<llama_seq_id*> sids(T_prefill);
|
|
std::vector<int8_t> lg(T_prefill, 0);
|
|
for (int i = 0; i < T_prefill; ++i) {
|
|
if (npe == 4) { pos[i] = i; pos[T_prefill + i] = i; pos[2*T_prefill + i] = i; pos[3*T_prefill + i] = 0; }
|
|
else { pos[i] = i; }
|
|
sids[i] = &sid0[i];
|
|
}
|
|
lg[T_prefill - 1] = 1;
|
|
llama_batch b{};
|
|
b.n_tokens = T_prefill; b.embd = prefill.data();
|
|
b.pos = pos.data(); b.n_seq_id = nsd.data(); b.seq_id = sids.data(); b.logits = lg.data();
|
|
if (llama_decode(ctx, b) != 0) { printf("prefill FAILED\n"); return 1; }
|
|
}
|
|
const double t_pfill = now_s() - t_pfill0;
|
|
|
|
// --- LOOP
|
|
// Sampler Talker (CB0) HF-style. Historique sur les CB0 émis (Talker = phrase entière).
|
|
Sampler talker_sampler{};
|
|
talker_sampler.temp = getenv("KZTTS_TEMP") ? (float)atof(getenv("KZTTS_TEMP")) : 0.9f;
|
|
talker_sampler.top_k = getenv("KZTTS_TOPK") ? atoi(getenv("KZTTS_TOPK")) : 50;
|
|
talker_sampler.top_p = getenv("KZTTS_TOPP") ? (float)atof(getenv("KZTTS_TOPP")) : 1.0f;
|
|
talker_sampler.rep_penalty = getenv("KZTTS_REPP") ? (float)atof(getenv("KZTTS_REPP")) : 1.05f;
|
|
talker_sampler.rep_window = 64;
|
|
sampler_seed(talker_sampler, SEED);
|
|
printf("Talker sampling: temp=%.2f top_k=%d top_p=%.2f rep_penalty=%.2f seed=%u\n",
|
|
talker_sampler.temp, talker_sampler.top_k, talker_sampler.top_p, talker_sampler.rep_penalty, SEED);
|
|
|
|
// logits du prefill : on les copie dans un buffer mutable (le sampler modifie en place)
|
|
std::vector<float> logits_buf(n_vocab);
|
|
{
|
|
const float* lp = llama_get_logits_ith(ctx, -1);
|
|
memcpy(logits_buf.data(), lp, n_vocab * sizeof(float));
|
|
}
|
|
int cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab);
|
|
std::vector<float> hidden_for_cp(n_embd);
|
|
{ const float* hh = llama_get_embeddings_ith(ctx, -1); if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float)); }
|
|
|
|
std::vector<int32_t> codes_engine;
|
|
codes_engine.reserve((size_t)max_steps_arg * 16);
|
|
int n_eos = -1;
|
|
double t_decode_total = 0, t_cp_total = 0;
|
|
const double t_loop0 = now_s();
|
|
int N_done = 0;
|
|
|
|
for (int s = 0; s < max_steps_arg; ++s) {
|
|
// CB0 = greedy
|
|
if (cb0 == codec_eos && n_eos < 0) { n_eos = s; printf(" step %d: EOS\n", s); break; }
|
|
codes_engine.push_back(cb0);
|
|
// CB1..15 via CP
|
|
const double tcp0 = now_s();
|
|
const float* cb0_emb = tok_embd.data() + (size_t)cb0 * n_embd;
|
|
int32_t cb15[15];
|
|
if (cp_use_cache) cp_predict_cached(cp_state, hidden_for_cp.data(), cb0_emb, cb15);
|
|
else cp_predict (cp_state, hidden_for_cp.data(), cb0_emb, cb15);
|
|
t_cp_total += now_s() - tcp0;
|
|
for (int i = 0; i < 15; ++i) codes_engine.push_back(cb15[i]);
|
|
|
|
// next_embed = sum 16 codecs + tts_pad
|
|
std::vector<float> next_embed(n_embd, 0.0f);
|
|
const float* e_cb0 = tok_embd.data() + (size_t)cb0 * n_embd;
|
|
for (int d = 0; d < n_embd; ++d) next_embed[d] = e_cb0[d];
|
|
for (int i = 1; i < 16; ++i) {
|
|
int code = cb15[i - 1];
|
|
const float* e = cp_state.codec_embs.data() + ((size_t)(i-1) * 2048 + code) * n_embd;
|
|
for (int d = 0; d < n_embd; ++d) next_embed[d] += e[d];
|
|
}
|
|
for (int d = 0; d < n_embd; ++d) next_embed[d] += tts_pad_emb[d];
|
|
|
|
// decode talker
|
|
llama_pos pos1[4] = {0,0,0,0};
|
|
const llama_pos p = T_prefill + s;
|
|
if (npe == 4) { pos1[0] = p; pos1[1] = p; pos1[2] = p; pos1[3] = 0; }
|
|
else { pos1[0] = p; }
|
|
int32_t nn = 1; llama_seq_id sd = 0; llama_seq_id* sp = &sd; int8_t l = 1;
|
|
llama_batch b{};
|
|
b.n_tokens = 1; b.embd = next_embed.data();
|
|
b.pos = pos1; b.n_seq_id = &nn; b.seq_id = &sp; b.logits = &l;
|
|
const double td0 = now_s();
|
|
if (llama_decode(ctx, b) != 0) { printf("step %d FAILED\n", s); break; }
|
|
t_decode_total += now_s() - td0;
|
|
|
|
{
|
|
const float* lp = llama_get_logits_ith(ctx, -1);
|
|
memcpy(logits_buf.data(), lp, n_vocab * sizeof(float));
|
|
}
|
|
cb0 = sampler_sample(talker_sampler, logits_buf.data(), n_vocab);
|
|
const float* hh = llama_get_embeddings_ith(ctx, -1);
|
|
if (hh) memcpy(hidden_for_cp.data(), hh, n_embd * sizeof(float));
|
|
N_done = s + 1;
|
|
}
|
|
const double t_loop = now_s() - t_loop0;
|
|
const int N = N_done;
|
|
const double audio_s = N / 12.0;
|
|
printf("=== TTS Talker+CP : N=%d frames (audio %.2fs) en %.3fs (RTF %.2f) ===\n",
|
|
N, audio_s, t_pfill + t_loop, (t_pfill + t_loop) / audio_s);
|
|
printf(" prefill %.3fs | loop %.3fs (talker=%.3f cp=%.3f)\n", t_pfill, t_loop, t_decode_total, t_cp_total);
|
|
printf(" per-step talker=%.1fms cp=%.1fms\n", t_decode_total * 1000.0 / N, t_cp_total * 1000.0 / N);
|
|
|
|
// dump codes for debug
|
|
{
|
|
std::ofstream f("/data/local/tmp/kz-engine/pipeline_codes.bin", std::ios::binary);
|
|
f.write((const char*)codes_engine.data(), codes_engine.size() * sizeof(int32_t));
|
|
}
|
|
printf("codes (CB0 trajectory only, %d frames):\n ", N);
|
|
for (int t = 0; t < N; ++t) {
|
|
printf("%d ", codes_engine[t * 16 + 0]);
|
|
if ((t + 1) % 16 == 0) printf("\n ");
|
|
}
|
|
printf("\n");
|
|
// ----- 7) Decoder ggml : codes [N, 16] (time-major) -> WAV
|
|
// Decoder veut codes_flat[16 * T] codebook-major (CB-fastest dans son forward).
|
|
// codes_engine = [N, 16] time-major -> transpose en [16, N] codebook-major.
|
|
std::vector<int32_t> codes_dec(16 * N);
|
|
for (int t = 0; t < N; ++t)
|
|
for (int c = 0; c < 16; ++c)
|
|
codes_dec[c * N + t] = codes_engine[t * 16 + c];
|
|
|
|
const double t_dec0 = now_s();
|
|
auto wav = dec.forward(codes_dec, N);
|
|
const double t_dec = now_s() - t_dec0;
|
|
printf("=== Decoder : %.3fs (RTF dec %.2f) -> %zu samples (%.2fs @24k)\n",
|
|
t_dec, t_dec / audio_s, wav.size(), wav.size() / 24000.0);
|
|
|
|
write_wav_pcm16_mono(OUT_WAV, wav.data(), wav.size(), 24000);
|
|
printf("=== TOTAL pipeline : %.3fs -> RTF %.2f ===\n",
|
|
t_pfill + t_loop + t_dec, (t_pfill + t_loop + t_dec) / audio_s);
|
|
printf("WAV -> %s\n", OUT_WAV);
|
|
|
|
cp_free(cp_state);
|
|
llama_free(ctx);
|
|
llama_model_free(m);
|
|
if (use_kz_tok) kz_tok_free(kz_tok);
|
|
return 0;
|
|
}
|