diff --git a/dist/BUG_pte_inapp_qnn_version_mismatch.md b/dist/BUG_pte_inapp_qnn_version_mismatch.md index 0681fb0..9ca5c74 100644 --- a/dist/BUG_pte_inapp_qnn_version_mismatch.md +++ b/dist/BUG_pte_inapp_qnn_version_mismatch.md @@ -98,5 +98,5 @@ Le `ship_runtime/` contient `qnn_llama_runner` + les 7 libs 2.42 : c'est l'oracl ## 7. Données - `.pte` validés (génèrent avec 2.42) : `qwen3_4b_seq1024/`, `qwen3_8b_seq512/`, `qwen2_5_7b_seq512/`, `kazeia-tablet-backup/llm/hybrid_llama_qnn_guard4b.pte`. -- Runtime de référence : `/opt/Kazeia/pte_prep_work/ship_runtime/` (QAIRT 2.42, `libQnnHtp` = AISW_VERSION 2.42.0). +- Runtime de référence : `dist/lib-pte/` (QAIRT 2.42, `libQnnHtp` = AISW_VERSION 2.42.0). - À fournir pour clore : `/proc//maps` du process app pendant un load 8B + liste `unzip -l` des `libQnn*` de l'APK. diff --git a/dist/LLM_INTEGRATION.md b/dist/LLM_INTEGRATION.md index 6bede59..7a9850a 100644 --- a/dist/LLM_INTEGRATION.md +++ b/dist/LLM_INTEGRATION.md @@ -72,9 +72,9 @@ void free(long handle) - `generateFromIds` : prefill sur IDs pré-tokenisés → decode greedy jusqu'à EOS (`<|im_end|>`=151645) ou `maxNewTokens`. Renvoie **uniquement les IDs générés**. - **Parité bit-exacte** avec le runner CLI (Qwen3-4B : tokens identiques, ~18.8 tok/s). - `NEEDED` : **uniquement `libqnn_executorch_backend.so`** + libs système (le reste est statique). -- Source : `pte_prep_work/jni/kazeia_pte_jni.cpp`. +- Source : `dist/lib-pte/kazeia_pte_jni.cpp`. -### 4.3 Set runtime QAIRT 2.42 — `pte_prep_work/ship_runtime/` +### 4.3 Set runtime QAIRT 2.42 — `dist/lib-pte/` Rebuildé contre **QAIRT 2.42** (⚠ **doit == la version QNN d'export des `.pte`**). 7 `.so`. --- @@ -85,8 +85,8 @@ Rebuildé contre **QAIRT 2.42** (⚠ **doit == la version QNN d'export des `.pte **Pour le NPU `.pte` (8 fichiers)** : | Fichier | Source | |---|---| -| `libkazeia_pte.so` | `pte_prep_work/jni/` | -| `libqnn_executorch_backend.so` | `pte_prep_work/ship_runtime/` | +| `libkazeia_pte.so` | `dist/lib-pte/` | +| `libqnn_executorch_backend.so` | `dist/lib-pte/` | | `libQnnHtp.so` · `libQnnSystem.so` · `libQnnHtpPrepare.so` | idem | | `libQnnHtpV79Stub.so` · `libQnnHtpV79Skel.so` · `libQnnHtpNetRunExtensions.so` | idem | @@ -180,9 +180,9 @@ speaker.release(); guard.release() ``` # Moteur natif + runtime .pte -/opt/Kazeia/pte_prep_work/jni/libkazeia_pte.so # wrapper JNI (95 Mo) -/opt/Kazeia/pte_prep_work/jni/kazeia_pte_jni.cpp # source -/opt/Kazeia/pte_prep_work/ship_runtime/ # 7 libs QAIRT 2.42 +dist/lib-pte/libkazeia_pte.so # wrapper JNI (95 Mo) +dist/lib-pte/kazeia_pte_jni.cpp # source +dist/lib-pte/ # 7 libs QAIRT 2.42 # Kotlin /opt/Kazeia-engine/dist/jni/LlmLoader.kt # chargeur + interface + 2 moteurs diff --git a/dist/jni/kazeia_pte_jni.cpp b/dist/jni/kazeia_pte_jni.cpp new file mode 100644 index 0000000..888b51d --- /dev/null +++ b/dist/jni/kazeia_pte_jni.cpp @@ -0,0 +1,316 @@ +/* + * Kazeia ExecuTorch+QNN LLM JNI bridge. + * + * Wraps example::Runner (the qnn_llama_runner CLI logic) into a JNI-callable + * shared library `libkazeia_pte.so` so the Kazeia Android app can load and run + * dense .pte LLMs (Qwen3-4B/8B, Qwen2.5-7B, Qwen3Guard-4B) on the Hexagon V79 + * NPU from Kotlin. + * + * Java class: com.kazeia.llm.ExecuTorchJni (library name: kazeia_pte) + * + * long load(String ptePath, String tokenizerPath, int seqLen, int evalMode) + * long[] generateFromIds(long handle, long[] promptIds, int maxNewTokens) + * long[] lastStats(long handle) // [prefillMs, decodeMs, nGenerated] + * void free(long handle) + * + * Also exposes plain extern "C" entrypoints (kz_pte_*) for an on-device test + * harness that drives the same code path without a JVM. + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using executorch::extension::Module; +using executorch::runtime::Error; + +namespace { + +// Type-erased wrapper so the opaque handle does not depend on the kv-bitwidth +// template parameter of example::Runner. +struct IRunnerHolder { + virtual ~IRunnerHolder() = default; + virtual Error load() = 0; + virtual Error generate_from_ids( + const std::vector& prompt_ids, + int32_t max_new_tokens, + std::vector& out_ids) = 0; + virtual int64_t last_prefill_ms() const = 0; + virtual int64_t last_decode_ms() const = 0; + virtual int64_t last_num_generated() const = 0; +}; + +template +struct RunnerHolder : public IRunnerHolder { + example::Runner runner; + RunnerHolder( + std::unique_ptr module, + const std::string& decoder_model_version, + const std::string& model_path, + const std::string& tokenizer_path, + int eval_mode, + bool shared_buffer) + : runner( + std::move(module), + decoder_model_version, + model_path, + tokenizer_path, + /*dump_logits_path=*/"", + /*performance_output_path=*/"/data/local/tmp/pte_jni_test/perf.txt", + /*temperature=*/0.0f, + eval_mode, + shared_buffer, + /*ngram=*/0, + /*window=*/0, + /*gcap=*/0, + /*tokenizer=*/nullptr, + /*attention_sink_rope_module=*/nullptr) {} + + Error load() override { + return runner.load(); + } + Error generate_from_ids( + const std::vector& prompt_ids, + int32_t max_new_tokens, + std::vector& out_ids) override { + return runner.generate_from_ids(prompt_ids, max_new_tokens, out_ids); + } + int64_t last_prefill_ms() const override { + return runner.last_prefill_ms(); + } + int64_t last_decode_ms() const override { + return runner.last_decode_ms(); + } + int64_t last_num_generated() const override { + return runner.last_num_generated(); + } +}; + +// Infer the decoder model version string the Runner constructor expects. +// All current Kazeia targets are Qwen; default to "qwen3" (EOS <|im_end|>), +// switch to "qwen2_5" if the path clearly names a 2.5 checkpoint. +std::string infer_decoder_model_version(const std::string& pte_path) { + auto contains = [&](const char* needle) { + return pte_path.find(needle) != std::string::npos; + }; + if (contains("qwen2_5") || contains("qwen2.5") || contains("q25") || + contains("Qwen2.5") || contains("qwen25")) { + return "qwen2_5"; + } + return "qwen3"; +} + +// Build a runner holder (load module, pick kv bitwidth, run load()). +// Returns nullptr on failure. +IRunnerHolder* create_runner( + const std::string& pte_path, + const std::string& tokenizer_path, + int /*seq_len*/, + int eval_mode, + bool shared_buffer) { + std::unique_ptr module = std::make_unique( + pte_path.c_str(), Module::LoadMode::MmapUseMlockIgnoreErrors); + + // Older models default to 8-bit KV IO; newer ones publish a width method. + example::KvBitWidth kv_bitwidth = example::KvBitWidth::kWidth8; + if (module->method_names().ok() && + module->method_names()->count("get_kv_io_bit_width") > 0) { + auto res = module->get("get_kv_io_bit_width"); + if (res.ok()) { + kv_bitwidth = static_cast( + res.get().toScalar().to()); + } + } + + std::string dmv = infer_decoder_model_version(pte_path); + + IRunnerHolder* holder = nullptr; + if (kv_bitwidth == example::KvBitWidth::kWidth16) { + holder = new RunnerHolder( + std::move(module), dmv, pte_path, tokenizer_path, eval_mode, + shared_buffer); + } else { + holder = new RunnerHolder( + std::move(module), dmv, pte_path, tokenizer_path, eval_mode, + shared_buffer); + } + + Error err = holder->load(); + if (err != Error::Ok) { + ET_LOG(Error, "kazeia_pte: runner.load() failed (%d)", (int)err); + delete holder; + return nullptr; + } + return holder; +} + +} // namespace + +// ---------------------------------------------------------------------------- +// Plain C entrypoints (for the on-device test harness; no JVM required). +// ---------------------------------------------------------------------------- +extern "C" { + +void* kz_pte_load( + const char* pte_path, + const char* tokenizer_path, + int seq_len, + int eval_mode) { + // shared_buffer=true mirrors the validated qnn_llama_runner invocation. + return create_runner(pte_path, tokenizer_path, seq_len, eval_mode, true); +} + +// Writes generated ids into out (caller-allocated, capacity out_cap). +// Returns the number of generated ids (>=0), or -1 on error. If the count +// exceeds out_cap, only out_cap ids are written but the true count is returned. +int kz_pte_generate_from_ids( + void* handle, + const uint64_t* prompt_ids, + int n_prompt, + int max_new_tokens, + uint64_t* out, + int out_cap) { + if (handle == nullptr || prompt_ids == nullptr || n_prompt <= 0) { + return -1; + } + auto* holder = reinterpret_cast(handle); + std::vector in(prompt_ids, prompt_ids + n_prompt); + std::vector gen; + Error err = holder->generate_from_ids(in, max_new_tokens, gen); + if (err != Error::Ok) { + return -1; + } + int n = (int)gen.size(); + int n_copy = (n < out_cap) ? n : out_cap; + if (out != nullptr && n_copy > 0) { + std::memcpy(out, gen.data(), n_copy * sizeof(uint64_t)); + } + return n; +} + +void kz_pte_last_stats(void* handle, int64_t out[3]) { + if (handle == nullptr) { + out[0] = out[1] = out[2] = 0; + return; + } + auto* holder = reinterpret_cast(handle); + out[0] = holder->last_prefill_ms(); + out[1] = holder->last_decode_ms(); + out[2] = holder->last_num_generated(); +} + +void kz_pte_free(void* handle) { + if (handle != nullptr) { + delete reinterpret_cast(handle); + } +} + +} // extern "C" + +// ---------------------------------------------------------------------------- +// JNI entrypoints: com.kazeia.llm.ExecuTorchJni +// ---------------------------------------------------------------------------- +extern "C" { + +JNIEXPORT jlong JNICALL +Java_com_kazeia_llm_ExecuTorchJni_load( + JNIEnv* env, + jclass /*clazz*/, + jstring ptePath, + jstring tokenizerPath, + jint seqLen, + jint evalMode) { + const char* pte = env->GetStringUTFChars(ptePath, nullptr); + const char* tok = env->GetStringUTFChars(tokenizerPath, nullptr); + std::string pte_s = pte ? pte : ""; + std::string tok_s = tok ? tok : ""; + if (pte) env->ReleaseStringUTFChars(ptePath, pte); + if (tok) env->ReleaseStringUTFChars(tokenizerPath, tok); + + IRunnerHolder* holder = + create_runner(pte_s, tok_s, (int)seqLen, (int)evalMode, /*shared=*/true); + return reinterpret_cast(holder); +} + +JNIEXPORT jlongArray JNICALL +Java_com_kazeia_llm_ExecuTorchJni_generateFromIds( + JNIEnv* env, + jclass /*clazz*/, + jlong handle, + jlongArray promptIds, + jint maxNewTokens) { + if (handle == 0 || promptIds == nullptr) { + return env->NewLongArray(0); + } + auto* holder = reinterpret_cast(handle); + + jsize n = env->GetArrayLength(promptIds); + std::vector in(n); + { + jlong* p = env->GetLongArrayElements(promptIds, nullptr); + for (jsize i = 0; i < n; ++i) { + in[i] = static_cast(p[i]); + } + env->ReleaseLongArrayElements(promptIds, p, JNI_ABORT); + } + + std::vector gen; + Error err = holder->generate_from_ids(in, (int)maxNewTokens, gen); + if (err != Error::Ok) { + return env->NewLongArray(0); + } + + jlongArray result = env->NewLongArray((jsize)gen.size()); + if (result == nullptr) { + return env->NewLongArray(0); + } + if (!gen.empty()) { + std::vector tmp(gen.size()); + for (size_t i = 0; i < gen.size(); ++i) { + tmp[i] = static_cast(gen[i]); + } + env->SetLongArrayRegion(result, 0, (jsize)tmp.size(), tmp.data()); + } + return result; +} + +JNIEXPORT jlongArray JNICALL +Java_com_kazeia_llm_ExecuTorchJni_lastStats( + JNIEnv* env, + jclass /*clazz*/, + jlong handle) { + jlongArray result = env->NewLongArray(3); + if (result == nullptr) { + return nullptr; + } + jlong vals[3] = {0, 0, 0}; + if (handle != 0) { + auto* holder = reinterpret_cast(handle); + vals[0] = holder->last_prefill_ms(); + vals[1] = holder->last_decode_ms(); + vals[2] = holder->last_num_generated(); + } + env->SetLongArrayRegion(result, 0, 3, vals); + return result; +} + +JNIEXPORT void JNICALL +Java_com_kazeia_llm_ExecuTorchJni_free( + JNIEnv* /*env*/, + jclass /*clazz*/, + jlong handle) { + if (handle != 0) { + delete reinterpret_cast(handle); + } +} + +} // extern "C" diff --git a/dist/lib-pte/libQnnHtp.so b/dist/lib-pte/libQnnHtp.so new file mode 100755 index 0000000..d898e03 Binary files /dev/null and b/dist/lib-pte/libQnnHtp.so differ diff --git a/dist/lib-pte/libQnnHtpNetRunExtensions.so b/dist/lib-pte/libQnnHtpNetRunExtensions.so new file mode 100755 index 0000000..aebdab0 Binary files /dev/null and b/dist/lib-pte/libQnnHtpNetRunExtensions.so differ diff --git a/dist/lib-pte/libQnnHtpPrepare.so b/dist/lib-pte/libQnnHtpPrepare.so new file mode 100755 index 0000000..de076ab Binary files /dev/null and b/dist/lib-pte/libQnnHtpPrepare.so differ diff --git a/dist/lib-pte/libQnnHtpV79Skel.so b/dist/lib-pte/libQnnHtpV79Skel.so new file mode 100755 index 0000000..9ef3944 Binary files /dev/null and b/dist/lib-pte/libQnnHtpV79Skel.so differ diff --git a/dist/lib-pte/libQnnHtpV79Stub.so b/dist/lib-pte/libQnnHtpV79Stub.so new file mode 100755 index 0000000..ee3b8ab Binary files /dev/null and b/dist/lib-pte/libQnnHtpV79Stub.so differ diff --git a/dist/lib-pte/libQnnSystem.so b/dist/lib-pte/libQnnSystem.so new file mode 100755 index 0000000..6a104aa Binary files /dev/null and b/dist/lib-pte/libQnnSystem.so differ diff --git a/dist/lib-pte/libkazeia_pte.so b/dist/lib-pte/libkazeia_pte.so new file mode 100755 index 0000000..66f6b8e Binary files /dev/null and b/dist/lib-pte/libkazeia_pte.so differ diff --git a/dist/lib-pte/libqnn_executorch_backend.so b/dist/lib-pte/libqnn_executorch_backend.so new file mode 100755 index 0000000..ee4f963 Binary files /dev/null and b/dist/lib-pte/libqnn_executorch_backend.so differ diff --git a/dist/lib-pte/qnn_llama_runner b/dist/lib-pte/qnn_llama_runner new file mode 100755 index 0000000..020fbf8 Binary files /dev/null and b/dist/lib-pte/qnn_llama_runner differ