diff --git a/dist/HANDOFF.md b/dist/HANDOFF.md index 9ca9a02..276cd03 100644 --- a/dist/HANDOFF.md +++ b/dist/HANDOFF.md @@ -12,15 +12,18 @@ GGUF, pas de `.pte`. STT reste ORT-QAIRT (inchangé). - **Kernel GDN = chantier clos** : son calcul n'est PAS le goulot (cf RAPPORT_RD §7). Ne pas y retoucher pour la perf. Decode au plafond BW CPU (~25 GB/s). -## État du bridge JNI (`jni/kazeia_engine_jni.cpp`) — OPTION C câblée (27/05) -Le JNI implémente **C** (prefill HTP / decode CPU), validé multi-tour (`jni/dual_ctx_mt.cpp` : -3 tours cohérents + mémoire conversationnelle OK, prefill HTP 103-144 t/s, decode CPU ~7-9). -- `load()` : pose `GGML_HEXAGON_GDN_PREFILL=1` + `GGML_HEXAGON_OPFILTER=SSM_CONV` (avant init backend), - charge le modèle **2×** : instance HTP (ngl99, device HTP0, t8) + instance CPU (ngl0, t4). ~4.7 GB. -- `generate()` : prefill du prompt sur HTP → `llama_state_seq_get/set_data` transfère le KV → decode sur CPU. - Sans état entre appels (l'app passe l'historique complet dans le prompt → re-prefill HTP, rapide). -- KV **f16**, flash_attn ON, thinking OFF (`` vide), greedy. API inchangée : `load/generate/reset/free`. -- `lib/libkazeia_engine.so` **à jour (option C, 37 KB, 4 symboles JNI, variante SHARED)** + `libggml-hexagon.so` +## État du bridge JNI (`jni/kazeia_engine_jni.cpp`) — OPTION C, validé end-to-end (27/05) +Le JNI implémente **C** (prefill HTP / decode CPU). **Validé bout-en-bout sur device** via `jni/test_jni_native.cpp` +(dlopen + JNIEnv mock → `load`/`generate`/`generateRaw`/`free` : sortie FR cohérente, mémoire conversationnelle OK) +et `jni/dual_ctx_mt.cpp` (3 tours, prefill HTP 103-144 t/s, decode CPU ~7-9). +- `load()` : pose `GGML_HEXAGON_GDN_PREFILL=1` (le fallback CPU du conv1d multi-token est dans le backend, pas + d'OPFILTER), charge le modèle **2×** : instance HTP (ngl99, device HTP0, t8) + instance CPU (ngl0, t4). ~4.7 GB. +- **API** : `generate(sys,usr,max)` (mono-tour) **OU `generateRaw(prompt,max)`** (multi-tour, prompt pré-formaté). + `EngineLlmEngine.kt` fournit **`ChatSession`** (accumule l'historique + construit le ChatML validé → l'app n'a + pas à connaître le template). Cœur : prefill HTP → `llama_state_seq_get/set_data` (transfert KV) → decode CPU. + Sans état entre appels (re-prefill de l'historique complet sur HTP, rapide). +- KV **f16**, flash_attn ON, thinking OFF (`` vide), greedy. API : `load/generate/generateRaw/reset/free`. +- `lib/libkazeia_engine.so` **à jour (option C, 5 symboles JNI, SHARED)** + `libggml-hexagon.so` (fix SSM_CONV) + `libggml-htp-v79.so` à jour. Prebuilts utilisables tels quels (NDK r27d) ; **rebuild depuis `jni/` dans ton build app recommandé** pour garantir l'ABI/STL (CMakeLists.txt fourni, linke llama+ggml+ggml-base+log). diff --git a/dist/jni/EngineLlmEngine.kt b/dist/jni/EngineLlmEngine.kt index 48b35d8..88b4a07 100644 --- a/dist/jni/EngineLlmEngine.kt +++ b/dist/jni/EngineLlmEngine.kt @@ -1,14 +1,48 @@ package com.kazeia.llm -// Remplace ExecuTorchLlmEngine. Mono-moteur Speaker+Thinker (budget thinking off). +// Remplace ExecuTorchLlmEngine. Option C: prefill HTP / decode CPU (géré côté natif). +// Mono-tour: generate(sys, usr). Multi-tour: ChatSession (gère l'historique + le template ChatML). + class EngineJni { external fun load(ggufPath: String, nCtx: Int): Long external fun generate(h: Long, sys: String, usr: String, maxTok: Int): String - external fun reset(h: Long); external fun free(h: Long) + external fun generateRaw(h: Long, prompt: String, maxTok: Int): String // prompt complet déjà formaté + external fun reset(h: Long) + external fun free(h: Long) companion object { init { System.loadLibrary("kazeia_engine") } } } -class EngineLlmEngine(model: String, ctx: Int = 2048) { - private val jni = EngineJni(); private val h = jni.load(model, ctx) - // Thinker: bullets; Speaker: SYS_KAZEIA + bullets. reset() entre tours, ou cache persistant. - fun generate(sys: String, usr: String, max: Int = 64) = jni.generate(h, sys, usr, max) - fun reset() = jni.reset(h); fun release() = jni.free(h) + +class EngineLlmEngine(model: String, ctx: Int = 4096) { + private val jni = EngineJni() + private val h = jni.load(model, ctx) + init { require(h != 0L) { "Kazeia-Engine: échec du chargement du modèle ($model)" } } + + // Mono-tour : system + un message user. + fun generate(sys: String, usr: String, max: Int = 96) = jni.generate(h, sys, usr, max) + // Multi-tour : prompt complet pré-formaté (voir ChatSession). + fun generateRaw(prompt: String, max: Int = 96) = jni.generateRaw(h, prompt, max) + + fun newChat(system: String) = ChatSession(this, system) + fun reset() = jni.reset(h) + fun release() = jni.free(h) +} + +// Conversation multi-tour : accumule l'historique et construit le ChatML Qwen3.5 + thinking-off. +// Structure identique à celle validée (jni/dual_ctx_mt.cpp) : mémoire conversationnelle correcte. +class ChatSession(private val engine: EngineLlmEngine, private val system: String) { + private val turns = StringBuilder() // "<|im_start|>role\ntext<|im_end|>\n" accumulés + + fun ask(user: String, max: Int = 96): String { + val prompt = buildString { + append("<|im_start|>system\n").append(system).append("<|im_end|>\n") + append(turns) + append("<|im_start|>user\n").append(user).append("<|im_end|>\n") + append("<|im_start|>assistant\n\n\n\n\n") // thinking OFF déterministe + } + val resp = engine.generateRaw(prompt, max) + turns.append("<|im_start|>user\n").append(user).append("<|im_end|>\n") + .append("<|im_start|>assistant\n").append(resp).append("<|im_end|>\n") + return resp + } + + fun clear() { turns.setLength(0) } } diff --git a/dist/jni/kazeia_engine_jni.cpp b/dist/jni/kazeia_engine_jni.cpp index 6255d93..469311f 100644 --- a/dist/jni/kazeia_engine_jni.cpp +++ b/dist/jni/kazeia_engine_jni.cpp @@ -1,8 +1,8 @@ // kazeia_engine_jni.cpp — bridge LLM Kazeia-Engine (llama.cpp fork ql + Hexagon). -// OPTION C: prefill NPU/HTP (ngl99, SSM_CONV forcé CPU via OPFILTER) -> transfert KV -> decode CPU (ngl0, t4+fa). -// prefill ~100-180 t/s (vs ~14 CPU), decode CPU ~7-10 (KV f16). 2 instances du modèle (~4.7GB). -// SSM_CONV HTP cassé en prefill -> OPFILTER=SSM_CONV (le route sur ggml-cpu). cf HANDOFF.md / PERF.md. -// generate() est sans état entre appels (l'app passe l'historique complet dans le prompt). STT reste ORT-QAIRT. +// OPTION C: prefill NPU/HTP (ngl99, SSM_CONV multi-token routé CPU par le backend) -> transfert KV -> +// decode CPU (ngl0, t4+fa, KV f16). prefill ~110-180 t/s (vs ~14 CPU), decode CPU ~7-10. 2 instances (~4.7GB). +// API: load -> generate(sys,usr) [mono-tour] OU generateRaw(prompt) [multi-tour, l'app/Kotlin formate] -> reset/free. +// Sans état entre appels (l'app passe l'historique complet dans le prompt -> re-prefill HTP). STT reste ORT-QAIRT. #include #include #include @@ -25,43 +25,8 @@ static llama_context* make_ctx(llama_model* m, int nctx, int nthreads) { return llama_init_from_model(m, cp); } -extern "C" JNIEXPORT jlong JNICALL -Java_com_kazeia_llm_EngineJni_load(JNIEnv* e, jobject, jstring path, jint nctx) { - const char* p = e->GetStringUTFChars(path, 0); - // Posé AVANT l'init du backend hexagon (lu au registre). GDN prefill sur HTP. - // (Plus besoin de GGML_HEXAGON_OPFILTER=SSM_CONV : le fallback CPU multi-token du conv1d - // est désormais intégré dans le backend, ggml_hexagon_supported_ssm_conv n_t>1.) - setenv("GGML_HEXAGON_GDN_PREFILL", "1", 1); - llama_backend_init(); - - // device HTP0 explicite pour l'instance prefill - static ggml_backend_dev_t devs[2] = { nullptr, nullptr }; - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - auto d = ggml_backend_dev_get(i); - if (!strcmp(ggml_backend_dev_name(d), "HTP0")) devs[0] = d; - } - auto mp_h = llama_model_default_params(); mp_h.n_gpu_layers = 99; if (devs[0]) mp_h.devices = devs; - auto m_h = llama_model_load_from_file(p, mp_h); - auto mp_c = llama_model_default_params(); mp_c.n_gpu_layers = 0; - auto m_c = llama_model_load_from_file(p, mp_c); - e->ReleaseStringUTFChars(path, p); - if (!m_h || !m_c) return 0; - - auto* k = new KEngine{ m_h, make_ctx(m_h, nctx, 8), // prefill t8 - m_c, make_ctx(m_c, nctx, 4), // decode t4 - llama_model_get_vocab(m_h), llama_sampler_init_greedy() }; - return (jlong) k; -} - -extern "C" JNIEXPORT jstring JNICALL -Java_com_kazeia_llm_EngineJni_generate(JNIEnv* e, jobject, jlong h, jstring sys, jstring usr, jint maxTok) { - auto* k = (KEngine*) h; - const char* sp = e->GetStringUTFChars(sys, 0); const char* up = e->GetStringUTFChars(usr, 0); - // ChatML Qwen3.5 + vide = thinking OFF déterministe - std::string p = "<|im_start|>system\n"; p += sp; p += "<|im_end|>\n<|im_start|>user\n"; p += up; - p += "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; - e->ReleaseStringUTFChars(sys, sp); e->ReleaseStringUTFChars(usr, up); - +// Cœur option C : prefill du prompt sur HTP -> transfert KV -> decode sur CPU. Renvoie le texte généré. +static std::string kengine_run(KEngine* k, const std::string& p, int maxTok) { int n = -llama_tokenize(k->v, p.c_str(), p.size(), nullptr, 0, true, true); std::vector t(n); llama_tokenize(k->v, p.c_str(), p.size(), t.data(), n, true, true); @@ -69,7 +34,7 @@ Java_com_kazeia_llm_EngineJni_generate(JNIEnv* e, jobject, jlong h, jstring sys, // PREFILL sur HTP (KV vidé -> historique complet re-prefillé à chaque appel) llama_memory_clear(llama_get_memory(k->c_h), true); llama_batch b = llama_batch_get_one(t.data(), n); - if (llama_decode(k->c_h, b) != 0) return e->NewStringUTF(""); + if (llama_decode(k->c_h, b) != 0) return std::string(); // transfert état KV HTP -> CPU size_t sz = llama_state_seq_get_size(k->c_h, 0); @@ -91,6 +56,54 @@ Java_com_kazeia_llm_EngineJni_generate(JNIEnv* e, jobject, jlong h, jstring sys, if (llama_decode(k->c_c, sb) != 0) break; pos++; id = llama_sampler_sample(k->s, k->c_c, -1); } + return out; +} + +extern "C" JNIEXPORT jlong JNICALL +Java_com_kazeia_llm_EngineJni_load(JNIEnv* e, jobject, jstring path, jint nctx) { + const char* p = e->GetStringUTFChars(path, 0); + // Posé AVANT l'init du backend hexagon. GDN prefill sur HTP. Le fallback CPU multi-token du + // conv1d (SSM_CONV) est intégré au backend (ggml_hexagon_supported_ssm_conv), pas d'OPFILTER. + setenv("GGML_HEXAGON_GDN_PREFILL", "1", 1); + llama_backend_init(); + + static ggml_backend_dev_t devs[2] = { nullptr, nullptr }; + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto d = ggml_backend_dev_get(i); + if (!strcmp(ggml_backend_dev_name(d), "HTP0")) devs[0] = d; + } + auto mp_h = llama_model_default_params(); mp_h.n_gpu_layers = 99; if (devs[0]) mp_h.devices = devs; + auto m_h = llama_model_load_from_file(p, mp_h); + auto mp_c = llama_model_default_params(); mp_c.n_gpu_layers = 0; + auto m_c = llama_model_load_from_file(p, mp_c); + e->ReleaseStringUTFChars(path, p); + if (!m_h || !m_c) return 0; + + auto* k = new KEngine{ m_h, make_ctx(m_h, nctx, 8), // prefill t8 + m_c, make_ctx(m_c, nctx, 4), // decode t4 + llama_model_get_vocab(m_h), llama_sampler_init_greedy() }; + return (jlong) k; +} + +// Mono-tour : construit le ChatML (system + 1 tour user) + thinking-off, puis infère. +extern "C" JNIEXPORT jstring JNICALL +Java_com_kazeia_llm_EngineJni_generate(JNIEnv* e, jobject, jlong h, jstring sys, jstring usr, jint maxTok) { + auto* k = (KEngine*) h; + const char* sp = e->GetStringUTFChars(sys, 0); const char* up = e->GetStringUTFChars(usr, 0); + std::string p = "<|im_start|>system\n"; p += sp; p += "<|im_end|>\n<|im_start|>user\n"; p += up; + p += "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + e->ReleaseStringUTFChars(sys, sp); e->ReleaseStringUTFChars(usr, up); + std::string out = kengine_run(k, p, maxTok); + return e->NewStringUTF(out.c_str()); +} + +// Multi-tour : l'app/Kotlin fournit le prompt complet déjà formaté (ChatML + historique + thinking-off). +extern "C" JNIEXPORT jstring JNICALL +Java_com_kazeia_llm_EngineJni_generateRaw(JNIEnv* e, jobject, jlong h, jstring prompt, jint maxTok) { + auto* k = (KEngine*) h; + const char* pp = e->GetStringUTFChars(prompt, 0); + std::string out = kengine_run(k, std::string(pp), maxTok); + e->ReleaseStringUTFChars(prompt, pp); return e->NewStringUTF(out.c_str()); } diff --git a/dist/jni/test_jni_native.cpp b/dist/jni/test_jni_native.cpp new file mode 100644 index 0000000..d35cba8 --- /dev/null +++ b/dist/jni/test_jni_native.cpp @@ -0,0 +1,58 @@ +// test_jni_native.cpp — teste libkazeia_engine.so bout-en-bout SANS JVM: +// dlopen + JNIEnv mock (GetStringUTFChars/Release/NewStringUTF) -> load / generateRaw multi-tour / free. +// Valide le marshalling JNI réel + le flux option C sur device. +// Build: NDK clang++ -Iinclude jni/test_jni_native.cpp -ldl -o test_jni_native +// Run: GGML_HEXAGON_GDN_PREFILL=1 LD_LIBRARY_PATH=lib ADSP_LIBRARY_PATH=lib ./test_jni_native lib/libkazeia_engine.so model.gguf +#include +#include +#include +#include +#include +#include + +static const char* GetStringUTFChars_impl(JNIEnv*, jstring s, jboolean* c){ if(c)*c=0; return (const char*)s; } +static void ReleaseStringUTFChars_impl(JNIEnv*, jstring, const char*){} +static jstring NewStringUTF_impl(JNIEnv*, const char* u){ return (jstring) strdup(u?u:""); } + +int main(int argc, char** argv){ + if(argc<3){ printf("usage: test_jni_native lib.so model.gguf\n"); return 1; } + void* lib = dlopen(argv[1], RTLD_NOW|RTLD_GLOBAL); + if(!lib){ printf("dlopen fail: %s\n", dlerror()); return 1; } + + auto fload = (jlong(*)(JNIEnv*,jobject,jstring,jint)) dlsym(lib, "Java_com_kazeia_llm_EngineJni_load"); + auto fraw = (jstring(*)(JNIEnv*,jobject,jlong,jstring,jint)) dlsym(lib, "Java_com_kazeia_llm_EngineJni_generateRaw"); + auto fgen = (jstring(*)(JNIEnv*,jobject,jlong,jstring,jstring,jint)) dlsym(lib, "Java_com_kazeia_llm_EngineJni_generate"); + auto ffree = (void(*)(JNIEnv*,jobject,jlong)) dlsym(lib, "Java_com_kazeia_llm_EngineJni_free"); + if(!fload||!fraw||!fgen||!ffree){ printf("dlsym fail (%p %p %p %p)\n",(void*)fload,(void*)fraw,(void*)fgen,(void*)ffree); return 1; } + + JNINativeInterface iface; memset(&iface,0,sizeof iface); + iface.GetStringUTFChars = GetStringUTFChars_impl; + iface.ReleaseStringUTFChars = ReleaseStringUTFChars_impl; + iface.NewStringUTF = NewStringUTF_impl; + const JNINativeInterface* pf = &iface; + JNIEnv* env = (JNIEnv*) &pf; + + jlong h = fload(env, nullptr, (jstring) argv[2], 4096); + printf("load -> h=%p\n", (void*)h); + if(!h){ printf("load fail\n"); return 1; } + + // mono-tour + const char* sys = "Tu es Kazeia, soutien psy bref en francais, tutoiement."; + const char* usr = "Je n'arrive plus a dormir, je rumine la nuit."; + jstring r1 = fgen(env, nullptr, h, (jstring) sys, (jstring) usr, 60); + printf("\n[generate mono-tour]\n%s\n", (const char*) r1); + + // multi-tour: prompt ChatML complet (system + 1 echange + nouvelle question memoire) + std::string p = + "<|im_start|>system\nTu es Kazeia, soutien psy, francais, tutoiement.<|im_end|>\n" + "<|im_start|>user\nJe m'appelle Marc et je dors mal.<|im_end|>\n" + "<|im_start|>assistant\nMarc, je comprends, le manque de sommeil est epuisant.<|im_end|>\n" + "<|im_start|>user\nRappelle-moi mon prenom ?<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n"; + jstring r2 = fraw(env, nullptr, h, (jstring) p.c_str(), 40); + printf("\n[generateRaw multi-tour, test memoire]\n%s\n", (const char*) r2); + + ffree(env, nullptr, h); + printf("\nfree OK\n"); + return 0; +} diff --git a/dist/lib/libkazeia_engine.so b/dist/lib/libkazeia_engine.so index 863c70f..3fa446f 100755 Binary files a/dist/lib/libkazeia_engine.so and b/dist/lib/libkazeia_engine.so differ