diff --git a/dist/HANDOFF.md b/dist/HANDOFF.md index ad8209a..c6b8302 100644 --- a/dist/HANDOFF.md +++ b/dist/HANDOFF.md @@ -12,12 +12,16 @@ GGUF, pas de `.pte`. STT reste ORT-QAIRT (inchangé). - **Kernel GDN = chantier clos** : son calcul n'est PAS le goulot (cf RAPPORT_RD §7). Ne pas y retoucher pour la perf. Decode au plafond BW CPU (~25 GB/s). -## État du bridge JNI (`jni/kazeia_engine_jni.cpp`) — ce qui est CÂBLÉ -Config déjà correcte et alignée : -- `n_threads=4`, `flash_attn=ENABLED`, `n_batch=512`, sampler **greedy**. -- Thinking OFF déterministe (ChatML + `` vide injecté) → pas de ramble. -- API : `load(path,nCtx) → generate(h,sys,usr,maxTok) → reset(h) / free(h)`. -- **`n_gpu_layers = 0` → tout CPU** (pas de HTP). Choix après le ping-pong ngl99 (decode 0.2 t/s). +## État du bridge JNI (`jni/kazeia_engine_jni.cpp`) — OPTION C câblée (27/05) +Le JNI implémente **C** (prefill HTP / decode CPU), validé multi-tour (`jni/dual_ctx_mt.cpp` : +3 tours cohérents + mémoire conversationnelle OK, prefill HTP 103-144 t/s, decode CPU ~7-9). +- `load()` : pose `GGML_HEXAGON_GDN_PREFILL=1` + `GGML_HEXAGON_OPFILTER=SSM_CONV` (avant init backend), + charge le modèle **2×** : instance HTP (ngl99, device HTP0, t8) + instance CPU (ngl0, t4). ~4.7 GB. +- `generate()` : prefill du prompt sur HTP → `llama_state_seq_get/set_data` transfère le KV → decode sur CPU. + Sans état entre appels (l'app passe l'historique complet dans le prompt → re-prefill HTP, rapide). +- KV **f16**, flash_attn ON, thinking OFF (`` vide), greedy. API inchangée : `load/generate/reset/free`. +- ⚠ **Rebuild `libkazeia_engine.so` requis** (le .so livré est l'ancienne version CPU-only/q8_0). + `CMakeLists.txt` inchangé (linke llama+ggml+ggml-base+log) ; compile+linke vérifié (4 symboles JNI OK). ## Décisions (vérifiées 27/05, batterie 90%, device froid) 1. **KV = f16 (RÉSOLU, corrigé dans le JNI).** Mesuré : decode q35-lmq4 f16=**10.9** vs q8_0=**6.5** @@ -40,11 +44,14 @@ Config déjà correcte et alignée : ## Perf (sains, 27/05, device froid) | | prefill | decode | RAM | |---|--:|--:|--:| -| q35-lmq4 — A: CPU-only (**config livrée**) | 14 (CPU t4) | **10.9** | 2.4 GB | -| q35-lmq4 — B: HTP+OPFILTER mono-ctx | **181** | 6.4 (HTP) | 2.4 GB +ION | -| q35-lmq4 — C: HTP-prefill / CPU-decode | **181** | **10.9** | 4.7 GB +ION | +| q35-lmq4 — **C: HTP-prefill / CPU-decode (CÂBLÉ JNI)** | **103-180** (HTP, monte avec ctx) | **~7-9** (CPU, profondeur) | 4.7 GB | +| q35-lmq4 — A: CPU-only | 14 | 10.9 (ctx court) | 2.4 GB | +| q35-lmq4 — B: HTP+OPFILTER mono-ctx | 181 | 6.4 (HTP) | 2.4 GB +ION | | q35-lmq4 — HTP sans OPFILTER | 189 *(sortie CASSÉE: SSM_CONV)* | — | — | +Multi-tour mesuré (C) : tour1 63tok→prefill 103/decode 9 ; tour3 268tok→prefill 144/decode 6.7 ; mémoire OK. +Decode CPU baisse avec la profondeur de contexte (normal). Un tour ~80 tok ≈ prefill 1-3 s + decode 9-12 s. + Decode = ce que l'utilisateur ressent (10.9, OK). Prefill CPU 14 t/s → prompt 200 tok ≈ 14 s ; garder le system prompt + l'historique courts. Le prefill HTP rapide existe mais sort du charabia (cf décision 2). diff --git a/dist/jni/dual_ctx_mt.cpp b/dist/jni/dual_ctx_mt.cpp new file mode 100644 index 0000000..6e72d7e --- /dev/null +++ b/dist/jni/dual_ctx_mt.cpp @@ -0,0 +1,91 @@ +// dual_ctx_mt.cpp — valide l'option C en MULTI-TOUR: par tour, re-prefill full history sur HTP +// (+OPFILTER=SSM_CONV) -> transfert KV -> decode CPU. Vérifie cohérence, mémoire conversationnelle, perf. +// Run: GGML_HEXAGON_OPFILTER=SSM_CONV GGML_HEXAGON_GDN_PREFILL=1 LD_LIBRARY_PATH=lib ./dual_ctx_mt model.gguf +#include +#include +#include +#include +#include +#include +#include +#include "llama.h" +#include "ggml-backend.h" + +static int64_t usec(){ struct timespec ts; clock_gettime(CLOCK_MONOTONIC,&ts); return (int64_t)ts.tv_sec*1000000+ts.tv_nsec/1000; } +static long rss_mb(){ FILE*f=fopen("/proc/self/status","r"); if(!f)return -1; char l[256]; long k=0; + while(fgets(l,sizeof l,f)) if(sscanf(l,"VmRSS: %ld kB",&k)==1) break; fclose(f); return k/1024; } + +struct Msg { const char* role; std::string text; }; + +static std::string build_prompt(const char* sys, const std::vector& hist, const std::string& user){ + std::string p = "<|im_start|>system\n"; p+=sys; p+="<|im_end|>\n"; + for(auto& m: hist){ p+="<|im_start|>"; p+=m.role; p+="\n"; p+=m.text; p+="<|im_end|>\n"; } + p += "<|im_start|>user\n"; p+=user; p+="<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + return p; +} + +int main(int argc, char** argv){ + if(argc<2){ printf("usage: dual_ctx_mt model.gguf\n"); return 1; } + llama_backend_init(); + + // device HTP0 explicite pour ctx_h + static ggml_backend_dev_t devs[2]={nullptr,nullptr}; + for(size_t i=0;i hist; + for(int k=0;k<3;k++){ + std::string prompt = build_prompt(sys, hist, users[k]); + int n=-llama_tokenize(vo,prompt.c_str(),prompt.size(),0,0,true,true); + std::vector t(n); llama_tokenize(vo,prompt.c_str(),prompt.size(),t.data(),n,true,true); + + // re-prefill full history sur HTP (KV vidé) + llama_memory_clear(llama_get_memory(ctx_h), true); + int64_t a=usec(); auto b=llama_batch_get_one(t.data(),n); + if(llama_decode(ctx_h,b)!=0){printf("prefill fail tour %d\n",k);return 1;} + int64_t bb=usec(); double pf=n*1e6/(double)(bb-a); + + // transfert KV HTP -> CPU + size_t sz=llama_state_seq_get_size(ctx_h,0); std::vector buf(sz); + llama_state_seq_get_data(ctx_h, buf.data(), sz, 0); + llama_memory_clear(llama_get_memory(ctx_c), true); + llama_state_seq_set_data(ctx_c, buf.data(), sz, 0); + + // decode sur CPU + llama_token id=llama_sampler_sample(s,ctx_h,-1); + std::string resp; int pos=n, ndec=0; int64_t d0=usec(); + for(int i=0;i<100;i++){ + char z[128]; int l=llama_token_to_piece(vo,id,z,sizeof z,0,true); + if(llama_vocab_is_eog(vo,id)) break; + if(l>0) resp.append(z,l); + llama_token tk=id; llama_pos p=pos; int32_t ns=1; llama_seq_id sd=0,*sp=&sd; int8_t lg=1; + llama_batch sb; memset(&sb,0,sizeof sb); sb.n_tokens=1; sb.token=&tk; sb.pos=&p; sb.n_seq_id=&ns; sb.seq_id=&sp; sb.logits=≶ + if(llama_decode(ctx_c,sb)!=0){fprintf(stderr,"decode fail t%d@%d\n",k,i);break;} + pos++; ndec++; id=llama_sampler_sample(s,ctx_c,-1); + } + int64_t d1=usec(); double dec=ndec*1e6/(double)(d1-d0); + printf("\n===== TOUR %d (prompt %d tok | prefill HTP %.1f t/s | decode CPU %.1f t/s) =====\n", k+1, n, pf, dec); + printf("USER : %s\n", users[k]); + printf("KAZEIA: %s\n", resp.c_str()); + hist.push_back({"user",users[k]}); hist.push_back({"assistant",resp}); + } + fprintf(stderr,"RSS peak : %ld MB\n", rss_mb()); + return 0; +} diff --git a/dist/jni/kazeia_engine_jni.cpp b/dist/jni/kazeia_engine_jni.cpp index 37f8521..10c82b0 100644 --- a/dist/jni/kazeia_engine_jni.cpp +++ b/dist/jni/kazeia_engine_jni.cpp @@ -1,55 +1,109 @@ -// kazeia_engine_jni.cpp — bridge LLM Kazeia-Engine (llama.cpp upstream + Hexagon). -// Speaker+Thinker, GGUF, prefill NPU (ngl99/HTP0) + decode CPU (t4+fa = 14t/s; t8=contention). -// thinking off via template applique cote Kotlin (/no_think) OU enable_thinking=false. -// STT reste ORT-QAIRT. Build: link libllama+libcommon+ggml*; jniLibs htp auto-V79. +// kazeia_engine_jni.cpp — bridge LLM Kazeia-Engine (llama.cpp fork ql + Hexagon). +// OPTION C: prefill NPU/HTP (ngl99, SSM_CONV forcé CPU via OPFILTER) -> transfert KV -> decode CPU (ngl0, t4+fa). +// prefill ~100-180 t/s (vs ~14 CPU), decode CPU ~7-10 (KV f16). 2 instances du modèle (~4.7GB). +// SSM_CONV HTP cassé en prefill -> OPFILTER=SSM_CONV (le route sur ggml-cpu). cf HANDOFF.md / PERF.md. +// generate() est sans état entre appels (l'app passe l'historique complet dans le prompt). STT reste ORT-QAIRT. #include +#include #include #include +#include #include "llama.h" +#include "ggml-backend.h" -struct KEngine { llama_model* m; llama_context* c; const llama_vocab* v; llama_sampler* s; }; +struct KEngine { + llama_model* m_h; llama_context* c_h; // prefill HTP + llama_model* m_c; llama_context* c_c; // decode CPU + const llama_vocab* v; llama_sampler* s; +}; + +static llama_context* make_ctx(llama_model* m, int nctx, int nthreads) { + auto cp = llama_context_default_params(); + cp.n_ctx = nctx; cp.n_batch = nctx; cp.n_threads = nthreads; + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; // t4+fa optimum decode + cp.type_k = GGML_TYPE_F16; cp.type_v = GGML_TYPE_F16; // KV f16 (q8_0 = -40% decode, mesuré) + return llama_init_from_model(m, cp); +} extern "C" JNIEXPORT jlong JNICALL Java_com_kazeia_llm_EngineJni_load(JNIEnv* e, jobject, jstring path, jint nctx) { const char* p = e->GetStringUTFChars(path, 0); + // Doit être posé AVANT l'init du backend hexagon (lu au registre). + setenv("GGML_HEXAGON_GDN_PREFILL", "1", 1); // GDN prefill sur HTP + setenv("GGML_HEXAGON_OPFILTER", "SSM_CONV", 1); // conv1d HTP cassé en prefill -> sur CPU llama_backend_init(); - auto mp = llama_model_default_params(); mp.n_gpu_layers = 0; // decode CPU pur 14t/s (ngl99=ping-pong 0.2) - auto m = llama_model_load_from_file(p, mp); e->ReleaseStringUTFChars(path, p); - if (!m) return 0; - auto cp = llama_context_default_params(); cp.n_ctx = nctx; cp.n_threads = 4; cp.n_batch = 512; - cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; // t4+fa optimum decode (t8=contention) - // KV f16 (PAS q8_0): mesuré 27/05 batterie pleine, q35-lmq4 decode f16=10.9 vs q8_0=6.5 (-40%, - // déquant KV en flash-attn coûte plus que le BW à court/moyen contexte; idem d=512). q8_0 ne - // servirait que la RAM KV en très long contexte. Repasser q8_0 seulement si OOM KV avéré. - cp.type_k = GGML_TYPE_F16; cp.type_v = GGML_TYPE_F16; - auto* k = new KEngine{m, llama_init_from_model(m, cp), llama_model_get_vocab(m), - llama_sampler_init_greedy()}; - return (jlong)k; + + // device HTP0 explicite pour l'instance prefill + static ggml_backend_dev_t devs[2] = { nullptr, nullptr }; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto d = ggml_backend_dev_get(i); + if (!strcmp(ggml_backend_dev_name(d), "HTP0")) devs[0] = d; + } + auto mp_h = llama_model_default_params(); mp_h.n_gpu_layers = 99; if (devs[0]) mp_h.devices = devs; + auto m_h = llama_model_load_from_file(p, mp_h); + auto mp_c = llama_model_default_params(); mp_c.n_gpu_layers = 0; + auto m_c = llama_model_load_from_file(p, mp_c); + e->ReleaseStringUTFChars(path, p); + if (!m_h || !m_c) return 0; + + auto* k = new KEngine{ m_h, make_ctx(m_h, nctx, 8), // prefill t8 + m_c, make_ctx(m_c, nctx, 4), // decode t4 + llama_model_get_vocab(m_h), llama_sampler_init_greedy() }; + return (jlong) k; } extern "C" JNIEXPORT jstring JNICALL Java_com_kazeia_llm_EngineJni_generate(JNIEnv* e, jobject, jlong h, jstring sys, jstring usr, jint maxTok) { - auto* k = (KEngine*)h; + auto* k = (KEngine*) h; const char* sp = e->GetStringUTFChars(sys, 0); const char* up = e->GetStringUTFChars(usr, 0); - // ChatML Qwen3.5 + vide = thinking OFF deterministe (sinon boucle infinie) - std::string p = "<|im_start|>system\n"; p+=sp; p+="<|im_end|>\n<|im_start|>user\n"; p+=up; + // ChatML Qwen3.5 + vide = thinking OFF déterministe + std::string p = "<|im_start|>system\n"; p += sp; p += "<|im_end|>\n<|im_start|>user\n"; p += up; p += "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; e->ReleaseStringUTFChars(sys, sp); e->ReleaseStringUTFChars(usr, up); + int n = -llama_tokenize(k->v, p.c_str(), p.size(), nullptr, 0, true, true); - std::vector t(n); llama_tokenize(k->v, p.c_str(), p.size(), t.data(), n, true, true); - llama_batch b = llama_batch_get_one(t.data(), n); // prefill batch -> HTP - std::string out; char buf[256]; - for (int i = 0; i < maxTok && llama_decode(k->c, b) == 0; ++i) { - llama_token id = llama_sampler_sample(k->s, k->c, -1); // decode -> CPU + std::vector t(n); + llama_tokenize(k->v, p.c_str(), p.size(), t.data(), n, true, true); + + // PREFILL sur HTP (KV vidé -> historique complet re-prefillé à chaque appel) + llama_memory_clear(llama_get_memory(k->c_h), true); + llama_batch b = llama_batch_get_one(t.data(), n); + if (llama_decode(k->c_h, b) != 0) return e->NewStringUTF(""); + + // transfert état KV HTP -> CPU + size_t sz = llama_state_seq_get_size(k->c_h, 0); + std::vector buf(sz); + llama_state_seq_get_data(k->c_h, buf.data(), sz, 0); + llama_memory_clear(llama_get_memory(k->c_c), true); + llama_state_seq_set_data(k->c_c, buf.data(), sz, 0); + + // DECODE sur CPU (1er token depuis les logits prefill HTP, suite sur CPU) + llama_token id = llama_sampler_sample(k->s, k->c_h, -1); + std::string out; char zbuf[256]; int pos = n; + for (int i = 0; i < maxTok; ++i) { if (llama_vocab_is_eog(k->v, id)) break; - int l = llama_token_to_piece(k->v, id, buf, sizeof buf, 0, true); - if (l > 0) out.append(buf, l); - b = llama_batch_get_one(&id, 1); + int l = llama_token_to_piece(k->v, id, zbuf, sizeof zbuf, 0, true); + if (l > 0) out.append(zbuf, l); + llama_token tok = id; llama_pos pp = pos; int32_t ns = 1; llama_seq_id sd = 0, *spd = &sd; int8_t lg = 1; + llama_batch sb; memset(&sb, 0, sizeof sb); + sb.n_tokens = 1; sb.token = &tok; sb.pos = &pp; sb.n_seq_id = &ns; sb.seq_id = &spd; sb.logits = ≶ + if (llama_decode(k->c_c, sb) != 0) break; + pos++; id = llama_sampler_sample(k->s, k->c_c, -1); } return e->NewStringUTF(out.c_str()); } extern "C" JNIEXPORT void JNICALL -Java_com_kazeia_llm_EngineJni_reset(JNIEnv*, jobject, jlong h){ llama_memory_clear(llama_get_memory(((KEngine*)h)->c), true); } +Java_com_kazeia_llm_EngineJni_reset(JNIEnv*, jobject, jlong h){ + auto* k = (KEngine*) h; + llama_memory_clear(llama_get_memory(k->c_h), true); + llama_memory_clear(llama_get_memory(k->c_c), true); +} extern "C" JNIEXPORT void JNICALL -Java_com_kazeia_llm_EngineJni_free(JNIEnv*, jobject, jlong h){ auto*k=(KEngine*)h; llama_sampler_free(k->s); llama_free(k->c); llama_model_free(k->m); delete k; } +Java_com_kazeia_llm_EngineJni_free(JNIEnv*, jobject, jlong h){ + auto* k = (KEngine*) h; + llama_sampler_free(k->s); + llama_free(k->c_h); llama_free(k->c_c); + llama_model_free(k->m_h); llama_model_free(k->m_c); + delete k; +}