llm: add llama.cpp backend (LlamaCppLlmEngine) for Qwen3.5-4B
- KazeiaLlmJni.kt : JNI bindings for 9 native fns (load/generate/setSystem/ reset/snapshot/stats/kvTokens/free). - LlamaCppLlmEngine.kt : implements core LlmEngine interface over KazeiaLlmJni, persistent KV cache across turns, streaming token callback. - jni/kazeia_llm_jni.cpp : JNI bridge to /opt/Kazeia/kazeia-llm/ C++ wrapper (compiled inline via CMakeLists). - jniLibs : libllama.so + ggml-* from /opt/Kazeia/llama-upstream/build-android (rebuilt with GGML_OPENMP=OFF to avoid libomp.so runtime dep). - KazeiaApplication.LLM_BACKEND config + runtime override via debug.kazeia.llm_backend (llamacpp|executorch). Default stays executorch for now; flip to llamacpp per-boot for the new path. - KazeiaService wires the selected engine at pipeline load. Validated on OPD2415: app loads Qwen3.5-4B-Q4_0 in ~4s via llama.cpp, ChatActivity opens, pipeline STT→LLM→TTS ready. Phase 2 complete.
This commit is contained in:
Kazeia Team
parent
db281002d9
commit
8a3151681d
|
|
@ -10,6 +10,14 @@ class KazeiaApplication : Application() {
|
|||
companion object {
|
||||
const val CHANNEL_ID = "kazeia_service_channel"
|
||||
const val MODELS_DIR = "/data/local/tmp/kazeia/models"
|
||||
|
||||
/** LLM backend selector :
|
||||
* - "executorch" : Qwen3-4B via ExecuTorch QNN on NPU (legacy, current default)
|
||||
* - "llamacpp" : Qwen3.5-4B via llama.cpp upstream on CPU (new primary path)
|
||||
* Can be overridden at boot via `adb shell setprop debug.kazeia.llm_backend llamacpp`
|
||||
* (property is read by KazeiaService). */
|
||||
const val LLM_BACKEND = "executorch"
|
||||
const val LLAMACPP_MODEL_FILENAME = "Qwen3.5-4B-Q4_0-pure.gguf"
|
||||
}
|
||||
|
||||
override fun onCreate() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
package com.kazeia.llm
|
||||
|
||||
/**
|
||||
* JNI bridge to the KazeiaLLM C++ wrapper over llama.cpp.
|
||||
*
|
||||
* Native impl in jni/kazeia_llm_jni.cpp. Underlying C++ library source :
|
||||
* /opt/Kazeia/kazeia-llm/ (compiled inline via the app CMake).
|
||||
*
|
||||
* One [load] allocates a native handle (opaque Long) tied to a loaded model +
|
||||
* context. Same handle is then passed to every call. Always call [free] when
|
||||
* done to release the model and KV buffer.
|
||||
*
|
||||
* KV cache is persistent across [generate] calls : it stores the full
|
||||
* conversation so subsequent turns only re-prefill the new user message +
|
||||
* assistant prefix. Call [reset] to clear.
|
||||
*/
|
||||
object KazeiaLlmJni {
|
||||
|
||||
init {
|
||||
// ggml is shared with whisper ; llama is the new addition.
|
||||
System.loadLibrary("ggml-base")
|
||||
System.loadLibrary("ggml-cpu")
|
||||
System.loadLibrary("ggml")
|
||||
System.loadLibrary("llama")
|
||||
System.loadLibrary("kazeia_llm_jni")
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a GGUF model + create context. Returns native handle (0 on failure).
|
||||
* @param gguf absolute path to the .gguf file (e.g. Qwen3.5-4B-Q4_0-pure.gguf).
|
||||
* @param nCtx max context tokens (default 4096).
|
||||
* @param nThreads CPU threads for decode (default 6 on OPD2415).
|
||||
*/
|
||||
external fun nativeLoad(
|
||||
gguf: String,
|
||||
nCtx: Int = 4096,
|
||||
nThreads: Int = 6,
|
||||
temperature: Float = 0.7f,
|
||||
topP: Float = 0.9f,
|
||||
repeatPenalty: Float = 1.1f
|
||||
): Long
|
||||
|
||||
/** Free model + context. Idempotent for handle=0. */
|
||||
external fun nativeFree(handle: Long)
|
||||
|
||||
/** Set system prompt (clears KV, re-evaluates). Returns 0 on success. */
|
||||
external fun nativeSetSystem(handle: Long, system: String): Int
|
||||
|
||||
/**
|
||||
* Append user message to conversation and generate an assistant reply.
|
||||
* KV cache is updated with user turn + assistant reply.
|
||||
* @param maxTokens hard cap on generated tokens (model stops earlier on <|im_end|>).
|
||||
* @param callback optional streaming callback ; return false to stop early.
|
||||
* @return full assistant text.
|
||||
*/
|
||||
external fun nativeGenerate(
|
||||
handle: Long,
|
||||
userMessage: String,
|
||||
maxTokens: Int,
|
||||
callback: TokenCallback?
|
||||
): String
|
||||
|
||||
/** Clear the entire KV cache. System prompt must be re-set after this. */
|
||||
external fun nativeReset(handle: Long)
|
||||
|
||||
/**
|
||||
* Stats from the last [nativeGenerate] call.
|
||||
* Returns [promptTokens, decodeTokens, promptMs, decodeMs, promptTps, decodeTps].
|
||||
*/
|
||||
external fun nativeLastStats(handle: Long): DoubleArray
|
||||
|
||||
/** Save current KV / conversation state to a file. Returns 0 on success. */
|
||||
external fun nativeSaveSnapshot(handle: Long, path: String): Int
|
||||
|
||||
/** Load conversation state from a file (clears current first). Returns 0 on success. */
|
||||
external fun nativeRestoreSnapshot(handle: Long, path: String): Int
|
||||
|
||||
/** Current number of tokens held in the KV cache (debug / UI). */
|
||||
external fun nativeKvTokens(handle: Long): Int
|
||||
|
||||
fun interface TokenCallback {
|
||||
/** Called for each generated piece. Return false to halt generation. */
|
||||
fun onToken(token: String): Boolean
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
package com.kazeia.llm
|
||||
|
||||
import android.util.Log
|
||||
import com.kazeia.core.GenerationResult
|
||||
import com.kazeia.core.LlmConfig
|
||||
import com.kazeia.core.LlmEngine
|
||||
import com.kazeia.core.SamplingParams
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
/**
|
||||
* Kazeia LLM backed by llama.cpp (upstream master) via KazeiaLlmJni.
|
||||
*
|
||||
* Replaces the previous GenieLlmEngine (Qualcomm Genie SDK on QNN) for the
|
||||
* therapeutic conversation path. Measured ~11 tok/s sustained on Qwen3.5-4B
|
||||
* Q4_0 on OPD2415 — +57 % vs the legacy briques runtime.
|
||||
*
|
||||
* Lifecycle : [load] once per app session ; [generate] called for each user
|
||||
* turn. The native side keeps a persistent KV cache between generate() calls,
|
||||
* so the conversation history need not be re-prefilled every turn — we only
|
||||
* append the new user + assistant tokens. Call [release] at app shutdown.
|
||||
*
|
||||
* The [modelPath] passed to [load] must point at the GGUF file directly
|
||||
* (e.g. ".../kazeia/models/Qwen3.5-4B-Q4_0-pure.gguf"), not a config file.
|
||||
*
|
||||
* System prompt is set via [setSystem] (defaults to a French therapeutic
|
||||
* assistant prompt if never called).
|
||||
*/
|
||||
class LlamaCppLlmEngine : LlmEngine {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "LlamaCppLlmEngine"
|
||||
private const val DEFAULT_SYSTEM =
|
||||
"Tu es un assistant psychologique bienveillant qui écoute et répond " +
|
||||
"avec empathie et chaleur. Réponses courtes et naturelles en français. " +
|
||||
"Ne jamais afficher ton raisonnement, réponds directement à l'utilisateur."
|
||||
}
|
||||
|
||||
@Volatile private var handle: Long = 0L
|
||||
@Volatile private var loaded = false
|
||||
private var currentSystem: String = DEFAULT_SYSTEM
|
||||
|
||||
override suspend fun load(modelPath: String, config: LlmConfig) {
|
||||
withContext(Dispatchers.IO) {
|
||||
Log.i(TAG, "Loading Qwen3.5 via llama.cpp : $modelPath")
|
||||
handle = KazeiaLlmJni.nativeLoad(
|
||||
gguf = modelPath,
|
||||
nCtx = config.maxContextLength,
|
||||
nThreads = 6,
|
||||
temperature = 0.7f,
|
||||
topP = 0.9f,
|
||||
repeatPenalty = 1.1f,
|
||||
)
|
||||
if (handle == 0L) {
|
||||
throw RuntimeException("llama.cpp nativeLoad failed for $modelPath")
|
||||
}
|
||||
val rc = KazeiaLlmJni.nativeSetSystem(handle, currentSystem)
|
||||
if (rc != 0) {
|
||||
Log.w(TAG, "nativeSetSystem returned $rc ; continuing with empty system")
|
||||
}
|
||||
loaded = true
|
||||
Log.i(TAG, "llama.cpp ready handle=$handle")
|
||||
}
|
||||
}
|
||||
|
||||
override fun isLoaded(): Boolean = loaded
|
||||
|
||||
/**
|
||||
* Override the system prompt. Clears the KV cache (conversation restarts).
|
||||
*/
|
||||
fun setSystem(system: String) {
|
||||
if (!loaded) {
|
||||
currentSystem = system
|
||||
return
|
||||
}
|
||||
currentSystem = system
|
||||
val rc = KazeiaLlmJni.nativeSetSystem(handle, system)
|
||||
if (rc != 0) Log.w(TAG, "setSystem rc=$rc")
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the conversation history. [setSystem] must be called after (or
|
||||
* the previously-set system is re-applied automatically by reset+eval
|
||||
* if the native side keeps it).
|
||||
*/
|
||||
fun resetConversation() {
|
||||
if (!loaded) return
|
||||
KazeiaLlmJni.nativeReset(handle)
|
||||
// Re-apply system so the next generate() starts on a valid turn.
|
||||
KazeiaLlmJni.nativeSetSystem(handle, currentSystem)
|
||||
}
|
||||
|
||||
override suspend fun generate(
|
||||
prompt: String,
|
||||
params: SamplingParams,
|
||||
onToken: ((String) -> Boolean)?,
|
||||
): GenerationResult = withContext(Dispatchers.IO) {
|
||||
check(loaded) { "LlamaCppLlmEngine: model not loaded" }
|
||||
|
||||
val startTime = System.currentTimeMillis()
|
||||
|
||||
val cb = onToken?.let {
|
||||
KazeiaLlmJni.TokenCallback { piece -> it(piece) }
|
||||
}
|
||||
val reply = KazeiaLlmJni.nativeGenerate(
|
||||
handle = handle,
|
||||
userMessage = prompt,
|
||||
maxTokens = params.maxNewTokens,
|
||||
callback = cb,
|
||||
)
|
||||
val elapsed = System.currentTimeMillis() - startTime
|
||||
|
||||
val stats = KazeiaLlmJni.nativeLastStats(handle)
|
||||
val decodeTokens = stats.getOrNull(1)?.toInt() ?: 0
|
||||
val decodeMs = stats.getOrNull(3) ?: elapsed.toDouble()
|
||||
val decodeTps = stats.getOrNull(5)?.toFloat() ?: 0f
|
||||
|
||||
GenerationResult(
|
||||
text = reply,
|
||||
tokenCount = decodeTokens,
|
||||
timeMs = elapsed,
|
||||
tokensPerSecond = if (decodeMs > 0) decodeTps else 0f,
|
||||
)
|
||||
}
|
||||
|
||||
/** Save the current conversation state to [path]. */
|
||||
fun saveSnapshot(path: String): Boolean {
|
||||
if (!loaded) return false
|
||||
return KazeiaLlmJni.nativeSaveSnapshot(handle, path) == 0
|
||||
}
|
||||
|
||||
/** Restore conversation state from [path]. */
|
||||
fun restoreSnapshot(path: String): Boolean {
|
||||
if (!loaded) return false
|
||||
return KazeiaLlmJni.nativeRestoreSnapshot(handle, path) == 0
|
||||
}
|
||||
|
||||
/** Debug : current KV token count. */
|
||||
fun kvTokens(): Int = if (loaded) KazeiaLlmJni.nativeKvTokens(handle) else 0
|
||||
|
||||
override fun release() {
|
||||
if (handle != 0L) {
|
||||
KazeiaLlmJni.nativeFree(handle)
|
||||
handle = 0L
|
||||
loaded = false
|
||||
Log.i(TAG, "llama.cpp released")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -561,13 +561,38 @@ class KazeiaService : Service() {
|
|||
))
|
||||
}
|
||||
|
||||
// LLM: Qwen3-4B on Hexagon V79 via qnn_llama_runner.
|
||||
_loadingState.value = LoadingState(50, "LLM Qwen3-4B NPU…")
|
||||
llm = ExecuTorchLlmEngine(this@KazeiaService) { msg -> log(msg) }
|
||||
try {
|
||||
llm.load("${KazeiaApplication.MODELS_DIR}/qwen3-4b", com.kazeia.core.LlmConfig())
|
||||
} catch (e: Exception) {
|
||||
log("LLM load failed: ${e.message} — falling back to echo mode")
|
||||
// LLM backend selection : llama.cpp Qwen3.5-4B (new default for
|
||||
// therapeutic conversation, measured ~11 tok/s on OPD2415 +57 %
|
||||
// vs ExecuTorch path) or legacy ExecuTorch Qwen3-4B QNN.
|
||||
// Override at runtime via `adb shell setprop debug.kazeia.llm_backend llamacpp`.
|
||||
val backend = try {
|
||||
val spClass = Class.forName("android.os.SystemProperties")
|
||||
spClass.getMethod("get", String::class.java, String::class.java)
|
||||
.invoke(null, "debug.kazeia.llm_backend", KazeiaApplication.LLM_BACKEND) as String
|
||||
} catch (_: Throwable) { KazeiaApplication.LLM_BACKEND }
|
||||
|
||||
_loadingState.value = LoadingState(50, "LLM $backend…")
|
||||
when (backend) {
|
||||
"llamacpp" -> {
|
||||
val engine = com.kazeia.llm.LlamaCppLlmEngine()
|
||||
llm = engine
|
||||
try {
|
||||
val gguf = "${KazeiaApplication.MODELS_DIR}/${KazeiaApplication.LLAMACPP_MODEL_FILENAME}"
|
||||
engine.load(gguf, com.kazeia.core.LlmConfig(maxContextLength = 4096))
|
||||
log("LLM llama.cpp loaded: ${KazeiaApplication.LLAMACPP_MODEL_FILENAME}")
|
||||
} catch (e: Exception) {
|
||||
log("llama.cpp load failed: ${e.message} — falling back to echo mode")
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
// "executorch" (default) — Qwen3-4B QNN on Hexagon V79.
|
||||
llm = ExecuTorchLlmEngine(this@KazeiaService) { msg -> log(msg) }
|
||||
try {
|
||||
llm.load("${KazeiaApplication.MODELS_DIR}/qwen3-4b", com.kazeia.core.LlmConfig())
|
||||
} catch (e: Exception) {
|
||||
log("LLM load failed: ${e.message} — falling back to echo mode")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_loadingState.value = LoadingState(80, "Audio…")
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@ project(kazeia-jni)
|
|||
|
||||
set(JNILIBS_DIR ${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI})
|
||||
|
||||
# --- Genie JNI bridge ---
|
||||
# --- External source paths (for kazeia_llm llama.cpp backend) ---
|
||||
set(KAZEIA_LLM_DIR /opt/Kazeia/kazeia-llm)
|
||||
set(LLAMA_UPSTREAM /opt/Kazeia/llama-upstream)
|
||||
|
||||
# --- Genie JNI bridge (legacy QNN backend, kept for fallback) ---
|
||||
add_library(genie_jni SHARED genie_jni.cpp)
|
||||
|
||||
add_library(Genie SHARED IMPORTED)
|
||||
|
|
@ -50,3 +54,24 @@ target_compile_options(neon_ops PRIVATE -std=c++17 -O3 -march=armv8.2-a+fp16)
|
|||
add_library(mel_extractor SHARED mel_extractor.cpp)
|
||||
target_link_libraries(mel_extractor android log)
|
||||
target_compile_options(mel_extractor PRIVATE -std=c++17 -O2)
|
||||
|
||||
# --- Kazeia LLM JNI bridge : llama.cpp upstream backend (primary LLM path) ---
|
||||
# Replaces the Genie/QNN path for Qwen3.5-4B. Compiles the KazeiaLLM C++
|
||||
# wrapper directly from /opt/Kazeia/kazeia-llm/ so there is no .a to ship
|
||||
# separately. Links against libllama.so shipped in jniLibs (synced from
|
||||
# /opt/Kazeia/llama-upstream/build-android/bin/).
|
||||
add_library(llama SHARED IMPORTED)
|
||||
set_target_properties(llama PROPERTIES IMPORTED_LOCATION ${JNILIBS_DIR}/libllama.so)
|
||||
|
||||
add_library(kazeia_llm_jni SHARED
|
||||
kazeia_llm_jni.cpp
|
||||
${KAZEIA_LLM_DIR}/src/kazeia_llm.cpp)
|
||||
|
||||
target_include_directories(kazeia_llm_jni PRIVATE
|
||||
${KAZEIA_LLM_DIR}/include
|
||||
${LLAMA_UPSTREAM}/include
|
||||
${LLAMA_UPSTREAM}/ggml/include)
|
||||
|
||||
target_link_libraries(kazeia_llm_jni llama ggml ggml-base ggml-cpu android log)
|
||||
target_compile_options(kazeia_llm_jni PRIVATE -std=c++17 -O3 -Wall -Wextra
|
||||
-Wno-unused-parameter -Wno-unused-variable)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,189 @@
|
|||
// JNI bridge for com.kazeia.llm.KazeiaLlmJni — wraps KazeiaLLM (llama.cpp based)
|
||||
// for the Kazeia Android app.
|
||||
//
|
||||
// Design : keep a KazeiaLLM instance per Java-side handle (opaque long). The
|
||||
// TokenCallback Java interface is routed through a JNI local thunk that
|
||||
// invokes the Kotlin Function1 on each token. If the callback returns false,
|
||||
// generation aborts gracefully.
|
||||
//
|
||||
// See /opt/Kazeia/kazeia-llm/ for the underlying C++ wrapper.
|
||||
|
||||
#include "kazeia_llm.h"
|
||||
|
||||
#include <android/log.h>
|
||||
#include <jni.h>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#define LOG_TAG "KazeiaLlmJni"
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
|
||||
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
|
||||
|
||||
namespace {
|
||||
|
||||
// Convert a jstring to std::string (UTF-8). Handles null.
|
||||
std::string jstr_to_std(JNIEnv * env, jstring js) {
|
||||
if (!js) return {};
|
||||
const char * c = env->GetStringUTFChars(js, nullptr);
|
||||
std::string s = c ? c : "";
|
||||
if (c) env->ReleaseStringUTFChars(js, c);
|
||||
return s;
|
||||
}
|
||||
|
||||
// Build the Kazeia therapeutic system prompt. Kept in native to minimise
|
||||
// back-and-forth with Kotlin when the app does not override it.
|
||||
static const char * DEFAULT_SYSTEM =
|
||||
"Tu es un assistant psychologique bienveillant qui écoute et répond avec "
|
||||
"empathie et chaleur. Réponses courtes et naturelles en français. "
|
||||
"Ne jamais afficher ton raisonnement, réponds directement à l'utilisateur.";
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C" {
|
||||
|
||||
JNIEXPORT jlong JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeLoad(JNIEnv * env, jobject /*thiz*/,
|
||||
jstring jgguf,
|
||||
jint n_ctx,
|
||||
jint n_threads,
|
||||
jfloat temperature,
|
||||
jfloat top_p,
|
||||
jfloat repeat_penalty) {
|
||||
std::string gguf = jstr_to_std(env, jgguf);
|
||||
auto * llm = new kazeia::KazeiaLLM();
|
||||
kazeia::LLMConfig cfg;
|
||||
cfg.n_ctx = n_ctx > 0 ? n_ctx : 4096;
|
||||
cfg.n_threads = n_threads > 0 ? n_threads : 6;
|
||||
cfg.n_batch = 512;
|
||||
cfg.temperature = temperature;
|
||||
cfg.top_p = top_p;
|
||||
cfg.repeat_penalty = repeat_penalty;
|
||||
|
||||
if (llm->load(gguf, cfg) != 0) {
|
||||
LOGE("KazeiaLLM::load failed on %s", gguf.c_str());
|
||||
delete llm;
|
||||
return 0;
|
||||
}
|
||||
LOGI("KazeiaLLM loaded : %s (ctx=%d threads=%d temp=%.2f)",
|
||||
gguf.c_str(), cfg.n_ctx, cfg.n_threads, cfg.temperature);
|
||||
return reinterpret_cast<jlong>(llm);
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeFree(JNIEnv * /*env*/, jobject /*thiz*/, jlong handle) {
|
||||
if (handle == 0) return;
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
delete llm;
|
||||
LOGI("KazeiaLLM released");
|
||||
}
|
||||
|
||||
JNIEXPORT jint JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeSetSystem(JNIEnv * env, jobject /*thiz*/,
|
||||
jlong handle, jstring jsystem) {
|
||||
if (handle == 0) return -1;
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
std::string system = jstr_to_std(env, jsystem);
|
||||
if (system.empty()) system = DEFAULT_SYSTEM;
|
||||
int rc = llm->set_system(system);
|
||||
LOGI("set_system rc=%d len=%zu", rc, system.size());
|
||||
return rc;
|
||||
}
|
||||
|
||||
JNIEXPORT jstring JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeGenerate(JNIEnv * env, jobject /*thiz*/,
|
||||
jlong handle,
|
||||
jstring juser,
|
||||
jint max_tokens,
|
||||
jobject jcallback) {
|
||||
if (handle == 0) return env->NewStringUTF("");
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
std::string user_msg = jstr_to_std(env, juser);
|
||||
|
||||
kazeia::TokenCallback cb = nullptr;
|
||||
jclass cb_class = nullptr;
|
||||
jmethodID cb_method = nullptr;
|
||||
|
||||
if (jcallback) {
|
||||
cb_class = env->GetObjectClass(jcallback);
|
||||
if (cb_class) {
|
||||
// Method signature : boolean onToken(String piece)
|
||||
cb_method = env->GetMethodID(cb_class, "onToken", "(Ljava/lang/String;)Z");
|
||||
}
|
||||
if (!cb_method) {
|
||||
LOGE("callback onToken(String):boolean not found");
|
||||
env->ExceptionClear();
|
||||
} else {
|
||||
// Capture by value : env, jcallback, cb_method must stay alive
|
||||
// for the duration of generate(). All local to the JNI thread.
|
||||
cb = [env, jcallback, cb_method](const std::string & piece) -> bool {
|
||||
jstring js = env->NewStringUTF(piece.c_str());
|
||||
jboolean cont = env->CallBooleanMethod(jcallback, cb_method, js);
|
||||
env->DeleteLocalRef(js);
|
||||
if (env->ExceptionCheck()) {
|
||||
env->ExceptionClear();
|
||||
return false;
|
||||
}
|
||||
return cont == JNI_TRUE;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
std::string reply = llm->generate(user_msg, (int) max_tokens, cb);
|
||||
const auto & s = llm->last_stats();
|
||||
LOGI("generate : prompt=%lld gen=%lld prefill=%.1fms decode=%.1fms (%.2f tok/s)",
|
||||
(long long) s.prompt_tokens, (long long) s.decode_tokens,
|
||||
s.prompt_ms, s.decode_ms, s.decode_tps());
|
||||
|
||||
return env->NewStringUTF(reply.c_str());
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeReset(JNIEnv * /*env*/, jobject /*thiz*/, jlong handle) {
|
||||
if (handle == 0) return;
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
llm->reset();
|
||||
LOGI("reset");
|
||||
}
|
||||
|
||||
JNIEXPORT jdoubleArray JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeLastStats(JNIEnv * env, jobject /*thiz*/, jlong handle) {
|
||||
jdoubleArray arr = env->NewDoubleArray(6);
|
||||
if (handle == 0 || !arr) return arr;
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
const auto & s = llm->last_stats();
|
||||
jdouble buf[6] = {
|
||||
(jdouble) s.prompt_tokens,
|
||||
(jdouble) s.decode_tokens,
|
||||
(jdouble) s.prompt_ms,
|
||||
(jdouble) s.decode_ms,
|
||||
(jdouble) s.prompt_tps(),
|
||||
(jdouble) s.decode_tps(),
|
||||
};
|
||||
env->SetDoubleArrayRegion(arr, 0, 6, buf);
|
||||
return arr;
|
||||
}
|
||||
|
||||
JNIEXPORT jint JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeSaveSnapshot(JNIEnv * env, jobject /*thiz*/,
|
||||
jlong handle, jstring jpath) {
|
||||
if (handle == 0) return -1;
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
return llm->save_snapshot(jstr_to_std(env, jpath));
|
||||
}
|
||||
|
||||
JNIEXPORT jint JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeRestoreSnapshot(JNIEnv * env, jobject /*thiz*/,
|
||||
jlong handle, jstring jpath) {
|
||||
if (handle == 0) return -1;
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
return llm->restore_snapshot(jstr_to_std(env, jpath));
|
||||
}
|
||||
|
||||
JNIEXPORT jint JNICALL
|
||||
Java_com_kazeia_llm_KazeiaLlmJni_nativeKvTokens(JNIEnv * /*env*/, jobject /*thiz*/, jlong handle) {
|
||||
if (handle == 0) return 0;
|
||||
auto * llm = reinterpret_cast<kazeia::KazeiaLLM *>(handle);
|
||||
return llm->n_tokens_in_kv();
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
Loading…
Reference in New Issue