diff --git a/kazeia-android/app/src/main/java/com/kazeia/KazeiaApplication.kt b/kazeia-android/app/src/main/java/com/kazeia/KazeiaApplication.kt index a707cbb..196e843 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/KazeiaApplication.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/KazeiaApplication.kt @@ -10,6 +10,14 @@ class KazeiaApplication : Application() { companion object { const val CHANNEL_ID = "kazeia_service_channel" const val MODELS_DIR = "/data/local/tmp/kazeia/models" + + /** LLM backend selector : + * - "executorch" : Qwen3-4B via ExecuTorch QNN on NPU (legacy, current default) + * - "llamacpp" : Qwen3.5-4B via llama.cpp upstream on CPU (new primary path) + * Can be overridden at boot via `adb shell setprop debug.kazeia.llm_backend llamacpp` + * (property is read by KazeiaService). */ + const val LLM_BACKEND = "executorch" + const val LLAMACPP_MODEL_FILENAME = "Qwen3.5-4B-Q4_0-pure.gguf" } override fun onCreate() { diff --git a/kazeia-android/app/src/main/java/com/kazeia/llm/KazeiaLlmJni.kt b/kazeia-android/app/src/main/java/com/kazeia/llm/KazeiaLlmJni.kt new file mode 100644 index 0000000..e8bce2f --- /dev/null +++ b/kazeia-android/app/src/main/java/com/kazeia/llm/KazeiaLlmJni.kt @@ -0,0 +1,85 @@ +package com.kazeia.llm + +/** + * JNI bridge to the KazeiaLLM C++ wrapper over llama.cpp. + * + * Native impl in jni/kazeia_llm_jni.cpp. Underlying C++ library source : + * /opt/Kazeia/kazeia-llm/ (compiled inline via the app CMake). + * + * One [load] allocates a native handle (opaque Long) tied to a loaded model + + * context. Same handle is then passed to every call. Always call [free] when + * done to release the model and KV buffer. + * + * KV cache is persistent across [generate] calls : it stores the full + * conversation so subsequent turns only re-prefill the new user message + + * assistant prefix. Call [reset] to clear. + */ +object KazeiaLlmJni { + + init { + // ggml is shared with whisper ; llama is the new addition. + System.loadLibrary("ggml-base") + System.loadLibrary("ggml-cpu") + System.loadLibrary("ggml") + System.loadLibrary("llama") + System.loadLibrary("kazeia_llm_jni") + } + + /** + * Load a GGUF model + create context. Returns native handle (0 on failure). + * @param gguf absolute path to the .gguf file (e.g. Qwen3.5-4B-Q4_0-pure.gguf). + * @param nCtx max context tokens (default 4096). + * @param nThreads CPU threads for decode (default 6 on OPD2415). + */ + external fun nativeLoad( + gguf: String, + nCtx: Int = 4096, + nThreads: Int = 6, + temperature: Float = 0.7f, + topP: Float = 0.9f, + repeatPenalty: Float = 1.1f + ): Long + + /** Free model + context. Idempotent for handle=0. */ + external fun nativeFree(handle: Long) + + /** Set system prompt (clears KV, re-evaluates). Returns 0 on success. */ + external fun nativeSetSystem(handle: Long, system: String): Int + + /** + * Append user message to conversation and generate an assistant reply. + * KV cache is updated with user turn + assistant reply. + * @param maxTokens hard cap on generated tokens (model stops earlier on <|im_end|>). + * @param callback optional streaming callback ; return false to stop early. + * @return full assistant text. + */ + external fun nativeGenerate( + handle: Long, + userMessage: String, + maxTokens: Int, + callback: TokenCallback? + ): String + + /** Clear the entire KV cache. System prompt must be re-set after this. */ + external fun nativeReset(handle: Long) + + /** + * Stats from the last [nativeGenerate] call. + * Returns [promptTokens, decodeTokens, promptMs, decodeMs, promptTps, decodeTps]. + */ + external fun nativeLastStats(handle: Long): DoubleArray + + /** Save current KV / conversation state to a file. Returns 0 on success. */ + external fun nativeSaveSnapshot(handle: Long, path: String): Int + + /** Load conversation state from a file (clears current first). Returns 0 on success. */ + external fun nativeRestoreSnapshot(handle: Long, path: String): Int + + /** Current number of tokens held in the KV cache (debug / UI). */ + external fun nativeKvTokens(handle: Long): Int + + fun interface TokenCallback { + /** Called for each generated piece. Return false to halt generation. */ + fun onToken(token: String): Boolean + } +} diff --git a/kazeia-android/app/src/main/java/com/kazeia/llm/LlamaCppLlmEngine.kt b/kazeia-android/app/src/main/java/com/kazeia/llm/LlamaCppLlmEngine.kt new file mode 100644 index 0000000..e50fd39 --- /dev/null +++ b/kazeia-android/app/src/main/java/com/kazeia/llm/LlamaCppLlmEngine.kt @@ -0,0 +1,149 @@ +package com.kazeia.llm + +import android.util.Log +import com.kazeia.core.GenerationResult +import com.kazeia.core.LlmConfig +import com.kazeia.core.LlmEngine +import com.kazeia.core.SamplingParams +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** + * Kazeia LLM backed by llama.cpp (upstream master) via KazeiaLlmJni. + * + * Replaces the previous GenieLlmEngine (Qualcomm Genie SDK on QNN) for the + * therapeutic conversation path. Measured ~11 tok/s sustained on Qwen3.5-4B + * Q4_0 on OPD2415 — +57 % vs the legacy briques runtime. + * + * Lifecycle : [load] once per app session ; [generate] called for each user + * turn. The native side keeps a persistent KV cache between generate() calls, + * so the conversation history need not be re-prefilled every turn — we only + * append the new user + assistant tokens. Call [release] at app shutdown. + * + * The [modelPath] passed to [load] must point at the GGUF file directly + * (e.g. ".../kazeia/models/Qwen3.5-4B-Q4_0-pure.gguf"), not a config file. + * + * System prompt is set via [setSystem] (defaults to a French therapeutic + * assistant prompt if never called). + */ +class LlamaCppLlmEngine : LlmEngine { + + companion object { + private const val TAG = "LlamaCppLlmEngine" + private const val DEFAULT_SYSTEM = + "Tu es un assistant psychologique bienveillant qui écoute et répond " + + "avec empathie et chaleur. Réponses courtes et naturelles en français. " + + "Ne jamais afficher ton raisonnement, réponds directement à l'utilisateur." + } + + @Volatile private var handle: Long = 0L + @Volatile private var loaded = false + private var currentSystem: String = DEFAULT_SYSTEM + + override suspend fun load(modelPath: String, config: LlmConfig) { + withContext(Dispatchers.IO) { + Log.i(TAG, "Loading Qwen3.5 via llama.cpp : $modelPath") + handle = KazeiaLlmJni.nativeLoad( + gguf = modelPath, + nCtx = config.maxContextLength, + nThreads = 6, + temperature = 0.7f, + topP = 0.9f, + repeatPenalty = 1.1f, + ) + if (handle == 0L) { + throw RuntimeException("llama.cpp nativeLoad failed for $modelPath") + } + val rc = KazeiaLlmJni.nativeSetSystem(handle, currentSystem) + if (rc != 0) { + Log.w(TAG, "nativeSetSystem returned $rc ; continuing with empty system") + } + loaded = true + Log.i(TAG, "llama.cpp ready handle=$handle") + } + } + + override fun isLoaded(): Boolean = loaded + + /** + * Override the system prompt. Clears the KV cache (conversation restarts). + */ + fun setSystem(system: String) { + if (!loaded) { + currentSystem = system + return + } + currentSystem = system + val rc = KazeiaLlmJni.nativeSetSystem(handle, system) + if (rc != 0) Log.w(TAG, "setSystem rc=$rc") + } + + /** + * Clear the conversation history. [setSystem] must be called after (or + * the previously-set system is re-applied automatically by reset+eval + * if the native side keeps it). + */ + fun resetConversation() { + if (!loaded) return + KazeiaLlmJni.nativeReset(handle) + // Re-apply system so the next generate() starts on a valid turn. + KazeiaLlmJni.nativeSetSystem(handle, currentSystem) + } + + override suspend fun generate( + prompt: String, + params: SamplingParams, + onToken: ((String) -> Boolean)?, + ): GenerationResult = withContext(Dispatchers.IO) { + check(loaded) { "LlamaCppLlmEngine: model not loaded" } + + val startTime = System.currentTimeMillis() + + val cb = onToken?.let { + KazeiaLlmJni.TokenCallback { piece -> it(piece) } + } + val reply = KazeiaLlmJni.nativeGenerate( + handle = handle, + userMessage = prompt, + maxTokens = params.maxNewTokens, + callback = cb, + ) + val elapsed = System.currentTimeMillis() - startTime + + val stats = KazeiaLlmJni.nativeLastStats(handle) + val decodeTokens = stats.getOrNull(1)?.toInt() ?: 0 + val decodeMs = stats.getOrNull(3) ?: elapsed.toDouble() + val decodeTps = stats.getOrNull(5)?.toFloat() ?: 0f + + GenerationResult( + text = reply, + tokenCount = decodeTokens, + timeMs = elapsed, + tokensPerSecond = if (decodeMs > 0) decodeTps else 0f, + ) + } + + /** Save the current conversation state to [path]. */ + fun saveSnapshot(path: String): Boolean { + if (!loaded) return false + return KazeiaLlmJni.nativeSaveSnapshot(handle, path) == 0 + } + + /** Restore conversation state from [path]. */ + fun restoreSnapshot(path: String): Boolean { + if (!loaded) return false + return KazeiaLlmJni.nativeRestoreSnapshot(handle, path) == 0 + } + + /** Debug : current KV token count. */ + fun kvTokens(): Int = if (loaded) KazeiaLlmJni.nativeKvTokens(handle) else 0 + + override fun release() { + if (handle != 0L) { + KazeiaLlmJni.nativeFree(handle) + handle = 0L + loaded = false + Log.i(TAG, "llama.cpp released") + } + } +} diff --git a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt index b52c0bc..f39e3e7 100644 --- a/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt +++ b/kazeia-android/app/src/main/java/com/kazeia/service/KazeiaService.kt @@ -561,13 +561,38 @@ class KazeiaService : Service() { )) } - // LLM: Qwen3-4B on Hexagon V79 via qnn_llama_runner. - _loadingState.value = LoadingState(50, "LLM Qwen3-4B NPU…") - llm = ExecuTorchLlmEngine(this@KazeiaService) { msg -> log(msg) } - try { - llm.load("${KazeiaApplication.MODELS_DIR}/qwen3-4b", com.kazeia.core.LlmConfig()) - } catch (e: Exception) { - log("LLM load failed: ${e.message} — falling back to echo mode") + // LLM backend selection : llama.cpp Qwen3.5-4B (new default for + // therapeutic conversation, measured ~11 tok/s on OPD2415 +57 % + // vs ExecuTorch path) or legacy ExecuTorch Qwen3-4B QNN. + // Override at runtime via `adb shell setprop debug.kazeia.llm_backend llamacpp`. + val backend = try { + val spClass = Class.forName("android.os.SystemProperties") + spClass.getMethod("get", String::class.java, String::class.java) + .invoke(null, "debug.kazeia.llm_backend", KazeiaApplication.LLM_BACKEND) as String + } catch (_: Throwable) { KazeiaApplication.LLM_BACKEND } + + _loadingState.value = LoadingState(50, "LLM $backend…") + when (backend) { + "llamacpp" -> { + val engine = com.kazeia.llm.LlamaCppLlmEngine() + llm = engine + try { + val gguf = "${KazeiaApplication.MODELS_DIR}/${KazeiaApplication.LLAMACPP_MODEL_FILENAME}" + engine.load(gguf, com.kazeia.core.LlmConfig(maxContextLength = 4096)) + log("LLM llama.cpp loaded: ${KazeiaApplication.LLAMACPP_MODEL_FILENAME}") + } catch (e: Exception) { + log("llama.cpp load failed: ${e.message} — falling back to echo mode") + } + } + else -> { + // "executorch" (default) — Qwen3-4B QNN on Hexagon V79. + llm = ExecuTorchLlmEngine(this@KazeiaService) { msg -> log(msg) } + try { + llm.load("${KazeiaApplication.MODELS_DIR}/qwen3-4b", com.kazeia.core.LlmConfig()) + } catch (e: Exception) { + log("LLM load failed: ${e.message} — falling back to echo mode") + } + } } _loadingState.value = LoadingState(80, "Audio…") diff --git a/kazeia-android/app/src/main/jni/CMakeLists.txt b/kazeia-android/app/src/main/jni/CMakeLists.txt index 3ad0378..e6678ce 100644 --- a/kazeia-android/app/src/main/jni/CMakeLists.txt +++ b/kazeia-android/app/src/main/jni/CMakeLists.txt @@ -3,7 +3,11 @@ project(kazeia-jni) set(JNILIBS_DIR ${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}) -# --- Genie JNI bridge --- +# --- External source paths (for kazeia_llm llama.cpp backend) --- +set(KAZEIA_LLM_DIR /opt/Kazeia/kazeia-llm) +set(LLAMA_UPSTREAM /opt/Kazeia/llama-upstream) + +# --- Genie JNI bridge (legacy QNN backend, kept for fallback) --- add_library(genie_jni SHARED genie_jni.cpp) add_library(Genie SHARED IMPORTED) @@ -50,3 +54,24 @@ target_compile_options(neon_ops PRIVATE -std=c++17 -O3 -march=armv8.2-a+fp16) add_library(mel_extractor SHARED mel_extractor.cpp) target_link_libraries(mel_extractor android log) target_compile_options(mel_extractor PRIVATE -std=c++17 -O2) + +# --- Kazeia LLM JNI bridge : llama.cpp upstream backend (primary LLM path) --- +# Replaces the Genie/QNN path for Qwen3.5-4B. Compiles the KazeiaLLM C++ +# wrapper directly from /opt/Kazeia/kazeia-llm/ so there is no .a to ship +# separately. Links against libllama.so shipped in jniLibs (synced from +# /opt/Kazeia/llama-upstream/build-android/bin/). +add_library(llama SHARED IMPORTED) +set_target_properties(llama PROPERTIES IMPORTED_LOCATION ${JNILIBS_DIR}/libllama.so) + +add_library(kazeia_llm_jni SHARED + kazeia_llm_jni.cpp + ${KAZEIA_LLM_DIR}/src/kazeia_llm.cpp) + +target_include_directories(kazeia_llm_jni PRIVATE + ${KAZEIA_LLM_DIR}/include + ${LLAMA_UPSTREAM}/include + ${LLAMA_UPSTREAM}/ggml/include) + +target_link_libraries(kazeia_llm_jni llama ggml ggml-base ggml-cpu android log) +target_compile_options(kazeia_llm_jni PRIVATE -std=c++17 -O3 -Wall -Wextra + -Wno-unused-parameter -Wno-unused-variable) diff --git a/kazeia-android/app/src/main/jni/kazeia_llm_jni.cpp b/kazeia-android/app/src/main/jni/kazeia_llm_jni.cpp new file mode 100644 index 0000000..9e6d5da --- /dev/null +++ b/kazeia-android/app/src/main/jni/kazeia_llm_jni.cpp @@ -0,0 +1,189 @@ +// JNI bridge for com.kazeia.llm.KazeiaLlmJni — wraps KazeiaLLM (llama.cpp based) +// for the Kazeia Android app. +// +// Design : keep a KazeiaLLM instance per Java-side handle (opaque long). The +// TokenCallback Java interface is routed through a JNI local thunk that +// invokes the Kotlin Function1 on each token. If the callback returns false, +// generation aborts gracefully. +// +// See /opt/Kazeia/kazeia-llm/ for the underlying C++ wrapper. + +#include "kazeia_llm.h" + +#include +#include +#include +#include + +#define LOG_TAG "KazeiaLlmJni" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) + +namespace { + +// Convert a jstring to std::string (UTF-8). Handles null. +std::string jstr_to_std(JNIEnv * env, jstring js) { + if (!js) return {}; + const char * c = env->GetStringUTFChars(js, nullptr); + std::string s = c ? c : ""; + if (c) env->ReleaseStringUTFChars(js, c); + return s; +} + +// Build the Kazeia therapeutic system prompt. Kept in native to minimise +// back-and-forth with Kotlin when the app does not override it. +static const char * DEFAULT_SYSTEM = + "Tu es un assistant psychologique bienveillant qui écoute et répond avec " + "empathie et chaleur. Réponses courtes et naturelles en français. " + "Ne jamais afficher ton raisonnement, réponds directement à l'utilisateur."; + +} // namespace + +extern "C" { + +JNIEXPORT jlong JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeLoad(JNIEnv * env, jobject /*thiz*/, + jstring jgguf, + jint n_ctx, + jint n_threads, + jfloat temperature, + jfloat top_p, + jfloat repeat_penalty) { + std::string gguf = jstr_to_std(env, jgguf); + auto * llm = new kazeia::KazeiaLLM(); + kazeia::LLMConfig cfg; + cfg.n_ctx = n_ctx > 0 ? n_ctx : 4096; + cfg.n_threads = n_threads > 0 ? n_threads : 6; + cfg.n_batch = 512; + cfg.temperature = temperature; + cfg.top_p = top_p; + cfg.repeat_penalty = repeat_penalty; + + if (llm->load(gguf, cfg) != 0) { + LOGE("KazeiaLLM::load failed on %s", gguf.c_str()); + delete llm; + return 0; + } + LOGI("KazeiaLLM loaded : %s (ctx=%d threads=%d temp=%.2f)", + gguf.c_str(), cfg.n_ctx, cfg.n_threads, cfg.temperature); + return reinterpret_cast(llm); +} + +JNIEXPORT void JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeFree(JNIEnv * /*env*/, jobject /*thiz*/, jlong handle) { + if (handle == 0) return; + auto * llm = reinterpret_cast(handle); + delete llm; + LOGI("KazeiaLLM released"); +} + +JNIEXPORT jint JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeSetSystem(JNIEnv * env, jobject /*thiz*/, + jlong handle, jstring jsystem) { + if (handle == 0) return -1; + auto * llm = reinterpret_cast(handle); + std::string system = jstr_to_std(env, jsystem); + if (system.empty()) system = DEFAULT_SYSTEM; + int rc = llm->set_system(system); + LOGI("set_system rc=%d len=%zu", rc, system.size()); + return rc; +} + +JNIEXPORT jstring JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeGenerate(JNIEnv * env, jobject /*thiz*/, + jlong handle, + jstring juser, + jint max_tokens, + jobject jcallback) { + if (handle == 0) return env->NewStringUTF(""); + auto * llm = reinterpret_cast(handle); + std::string user_msg = jstr_to_std(env, juser); + + kazeia::TokenCallback cb = nullptr; + jclass cb_class = nullptr; + jmethodID cb_method = nullptr; + + if (jcallback) { + cb_class = env->GetObjectClass(jcallback); + if (cb_class) { + // Method signature : boolean onToken(String piece) + cb_method = env->GetMethodID(cb_class, "onToken", "(Ljava/lang/String;)Z"); + } + if (!cb_method) { + LOGE("callback onToken(String):boolean not found"); + env->ExceptionClear(); + } else { + // Capture by value : env, jcallback, cb_method must stay alive + // for the duration of generate(). All local to the JNI thread. + cb = [env, jcallback, cb_method](const std::string & piece) -> bool { + jstring js = env->NewStringUTF(piece.c_str()); + jboolean cont = env->CallBooleanMethod(jcallback, cb_method, js); + env->DeleteLocalRef(js); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + return false; + } + return cont == JNI_TRUE; + }; + } + } + + std::string reply = llm->generate(user_msg, (int) max_tokens, cb); + const auto & s = llm->last_stats(); + LOGI("generate : prompt=%lld gen=%lld prefill=%.1fms decode=%.1fms (%.2f tok/s)", + (long long) s.prompt_tokens, (long long) s.decode_tokens, + s.prompt_ms, s.decode_ms, s.decode_tps()); + + return env->NewStringUTF(reply.c_str()); +} + +JNIEXPORT void JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeReset(JNIEnv * /*env*/, jobject /*thiz*/, jlong handle) { + if (handle == 0) return; + auto * llm = reinterpret_cast(handle); + llm->reset(); + LOGI("reset"); +} + +JNIEXPORT jdoubleArray JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeLastStats(JNIEnv * env, jobject /*thiz*/, jlong handle) { + jdoubleArray arr = env->NewDoubleArray(6); + if (handle == 0 || !arr) return arr; + auto * llm = reinterpret_cast(handle); + const auto & s = llm->last_stats(); + jdouble buf[6] = { + (jdouble) s.prompt_tokens, + (jdouble) s.decode_tokens, + (jdouble) s.prompt_ms, + (jdouble) s.decode_ms, + (jdouble) s.prompt_tps(), + (jdouble) s.decode_tps(), + }; + env->SetDoubleArrayRegion(arr, 0, 6, buf); + return arr; +} + +JNIEXPORT jint JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeSaveSnapshot(JNIEnv * env, jobject /*thiz*/, + jlong handle, jstring jpath) { + if (handle == 0) return -1; + auto * llm = reinterpret_cast(handle); + return llm->save_snapshot(jstr_to_std(env, jpath)); +} + +JNIEXPORT jint JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeRestoreSnapshot(JNIEnv * env, jobject /*thiz*/, + jlong handle, jstring jpath) { + if (handle == 0) return -1; + auto * llm = reinterpret_cast(handle); + return llm->restore_snapshot(jstr_to_std(env, jpath)); +} + +JNIEXPORT jint JNICALL +Java_com_kazeia_llm_KazeiaLlmJni_nativeKvTokens(JNIEnv * /*env*/, jobject /*thiz*/, jlong handle) { + if (handle == 0) return 0; + auto * llm = reinterpret_cast(handle); + return llm->n_tokens_in_kv(); +} + +} // extern "C"