refonte(llm): moteur LLM unifié GGUF + .pte NPU (UnifiedLlmAdapter)
Remplace la couche LLM par la dernière version de kazeia-engine
(cf dist/LLM_INTEGRATION.md). Chargement par magic-byte :
- GGUF ("GGUF"@0) -> EngineLlmEngine (libllama CPU i8mm, session cache-préfixe KV)
- .pte ("ET12"@4) -> PteLlmEngine (ExecuTorch+QNN, NPU V79, sans root)
- LlmLoader.kt copié de l'engine + fix encode DJL (encode(p,false,false))
- UnifiedLlmAdapter : pont com.kazeia.core.LlmEngine, generateWithSystem par tour
- ModelRegistry : 5 modèles (Qwen3.5-4B GGUF défaut + 4 .pte en sous-dossiers)
- KazeiaService recâblé ; DEFAULT_SYSTEM_PROMPT -> ConfigStore.DEFAULT_SPEAKER_PROMPT
- DJL tokenizer arm64 Android : tokenizers + tokenizer-native alignés 0.33.0
(natif libdjl_tokenizer.so dans l'AAR ; natifs desktop exclus, -23 Mo APK)
- jniLibs : +libkazeia_pte.so, +libQnnHtpNetRunExtensions.so,
libqnn_executorch_backend.so -> QAIRT 2.42 ; manifest re-figé (28 libs)
- fichiers morts archivés (ExecuTorchLlmEngine, GenieLlmEngine)
Phase 1 (#287) : build debug green. Phase 2 = validation device.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kazeia Team
parent
3b58fcb467
commit
0a72654093
|
|
@ -106,6 +106,17 @@ android {
|
|||
jniLibs.srcDirs("src/main/jniLibs")
|
||||
}
|
||||
}
|
||||
|
||||
packaging {
|
||||
resources {
|
||||
// DJL `tokenizers` embarque des natifs desktop (macOS/Windows) inutiles
|
||||
// sur Android (~23 Mo) ; seul l'arm64 de `tokenizer-native` sert. On les
|
||||
// exclut pour alléger l'APK (payload self-update OTA).
|
||||
excludes += "native/lib/osx-**"
|
||||
excludes += "native/lib/win-**"
|
||||
excludes += "native/lib/linux-**"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reproductibilité des jniLibs : les .so (gitignorés, 5 chaînes de build) doivent
|
||||
|
|
@ -151,6 +162,14 @@ dependencies {
|
|||
implementation("com.facebook.soloader:nativeloader:0.10.5")
|
||||
implementation(files("libs/executorch.jar"))
|
||||
|
||||
// Tokeniseur HF (DJL) — pour PteLlmEngine (.pte NPU) : tokenise le ChatML Qwen
|
||||
// côté Kotlin puis passe les IDs au natif (tokeniseur natif de l'engine cassé pour Qwen).
|
||||
// L'artefact `tokenizers` ne porte QUE les natifs desktop (osx/win) ; le natif Android
|
||||
// arm64 (libdjl_tokenizer.so) vit dans l'AAR `ai.djl.android:tokenizer-native`, dont la
|
||||
// seule version publiée est 0.33.0 → on aligne les DEUX sur 0.33.0 (DJL exige l'égalité).
|
||||
implementation("ai.djl.huggingface:tokenizers:0.33.0")
|
||||
implementation("ai.djl.android:tokenizer-native:0.33.0")
|
||||
|
||||
// OkHttp — client WebDAV pour téléchargement modèles depuis Nextcloud
|
||||
implementation("com.squareup.okhttp3:okhttp:4.12.0")
|
||||
// WorkManager — orchestration du téléchargement modèles (foreground, reprise)
|
||||
|
|
|
|||
|
|
@ -54,27 +54,11 @@ object ModelRegistry {
|
|||
}
|
||||
}
|
||||
|
||||
// Catalogue refonte 2026-06-17 (cf dist/LLM_INTEGRATION.md §2). Chargement unifié
|
||||
// via UnifiedLlmAdapter (magic-byte) : GGUF→CPU i8mm, .pte→NPU V79 (libkazeia_pte).
|
||||
// .pte en SOUS-DOSSIER <id>/hybrid_llama_qnn.pte (+ tokenizer.json) : le nom du
|
||||
// dossier porte l'auto-détection decoder_model_version (qwen3/qwen2_5) côté natif.
|
||||
val ALL: List<ModelInfo> = listOf(
|
||||
ModelInfo(
|
||||
id = "qwen3-4b-seq512",
|
||||
displayName = "Qwen3-4B (seq=512)",
|
||||
pteFile = "hybrid_llama_qnn_4b.pte",
|
||||
tokenizerFile = "tokenizer.json",
|
||||
role = Role.SPEAKER,
|
||||
sizeMb = 3000,
|
||||
maxSeqLen = 512,
|
||||
notes = "Speaker historique, budget contexte serré"
|
||||
),
|
||||
ModelInfo(
|
||||
id = "qwen3-4b-seq1024",
|
||||
displayName = "Qwen3-4B (seq=1024) ★",
|
||||
pteFile = "hybrid_llama_qnn_4b_seq1024.pte",
|
||||
tokenizerFile = "tokenizer.json",
|
||||
role = Role.SPEAKER,
|
||||
sizeMb = 3000,
|
||||
maxSeqLen = 1024,
|
||||
notes = "Speaker recommandé — production stable 2026-05-14"
|
||||
),
|
||||
ModelInfo(
|
||||
id = "qwen3.5-4b",
|
||||
displayName = "Qwen3.5-4B (GGUF) ★",
|
||||
|
|
@ -83,43 +67,58 @@ object ModelRegistry {
|
|||
role = Role.SPEAKER,
|
||||
sizeMb = 2380,
|
||||
maxSeqLen = 4096,
|
||||
notes = "Speaker GGUF par défaut du moteur lib (q35-lmq4). Thinking-off Qwen3.5.",
|
||||
notes = "Speaker PAR DÉFAUT — hybride GatedDeltaNet (non exportable .pte), GGUF CPU i8mm. Thinking-off.",
|
||||
backend = Backend.GGUF,
|
||||
ggufFile = "q35-lmq4.gguf",
|
||||
chatTemplate = ChatTemplate.QWEN35_THINKOFF
|
||||
),
|
||||
ModelInfo(
|
||||
id = "qwen2.5-7b-instruct",
|
||||
displayName = "Qwen2.5-7B Instruct (GGUF)",
|
||||
pteFile = "",
|
||||
tokenizerFile = "",
|
||||
id = "qwen3-4b",
|
||||
displayName = "Qwen3-4B (.pte NPU)",
|
||||
pteFile = "qwen3_4b_seq1024/hybrid_llama_qnn.pte",
|
||||
tokenizerFile = "qwen3_4b_seq1024/tokenizer.json",
|
||||
role = Role.SPEAKER,
|
||||
sizeMb = 4683,
|
||||
maxSeqLen = 4096,
|
||||
notes = "Speaker GGUF dense (arch qwen2) via moteur lib CPU. Q4_K_M ~4.7 GB.",
|
||||
backend = Backend.GGUF,
|
||||
ggufFile = "Qwen2.5-7B-Instruct-Q4_K_M.gguf",
|
||||
sizeMb = 3100,
|
||||
maxSeqLen = 1024,
|
||||
notes = "Dense Qwen3-4B sur NPU. Prefill 313 tok/s, RAM ÷2, sans root. Decode 15.7.",
|
||||
backend = Backend.PTE,
|
||||
chatTemplate = ChatTemplate.QWEN35_THINKOFF
|
||||
),
|
||||
ModelInfo(
|
||||
id = "qwen3-8b",
|
||||
displayName = "Qwen3-8B (.pte NPU)",
|
||||
pteFile = "qwen3_8b_seq512/hybrid_llama_qnn.pte",
|
||||
tokenizerFile = "qwen3_8b_seq512/tokenizer.json",
|
||||
role = Role.SPEAKER,
|
||||
sizeMb = 5500,
|
||||
maxSeqLen = 512,
|
||||
notes = "Dense Qwen3-8B sur NPU. Decode 10.5, prefill 219, RAM ~3.1 G.",
|
||||
backend = Backend.PTE,
|
||||
chatTemplate = ChatTemplate.QWEN35_THINKOFF
|
||||
),
|
||||
ModelInfo(
|
||||
id = "qwen2.5-7b",
|
||||
displayName = "Qwen2.5-7B (.pte NPU)",
|
||||
pteFile = "qwen2_5_7b_seq512/hybrid_llama_qnn.pte",
|
||||
tokenizerFile = "qwen2_5_7b_seq512/tokenizer.json",
|
||||
role = Role.SPEAKER,
|
||||
sizeMb = 5000,
|
||||
maxSeqLen = 512,
|
||||
notes = "Dense Qwen2.5-7B sur NPU (decoder qwen2_5). Decode 8.0 (embeddings non-tied), RAM ~5.1 G.",
|
||||
backend = Backend.PTE,
|
||||
chatTemplate = ChatTemplate.PLAIN_CHATML
|
||||
),
|
||||
ModelInfo(
|
||||
id = "guard06b",
|
||||
displayName = "Qwen3Guard 0.6B",
|
||||
pteFile = "hybrid_llama_qnn_guard06b.pte",
|
||||
tokenizerFile = "tokenizer_guard06b.json",
|
||||
role = Role.THINKER,
|
||||
sizeMb = 672,
|
||||
maxSeqLen = 512,
|
||||
notes = "Thinker léger, ION ~700 MB, cascade safe"
|
||||
),
|
||||
ModelInfo(
|
||||
id = "guard4b",
|
||||
displayName = "Qwen3Guard 4B",
|
||||
pteFile = "hybrid_llama_qnn_guard4b.pte",
|
||||
tokenizerFile = "tokenizer_guard.json",
|
||||
displayName = "Qwen3Guard 4B (.pte NPU)",
|
||||
pteFile = "guard4b/hybrid_llama_qnn.pte",
|
||||
tokenizerFile = "guard4b/tokenizer.json",
|
||||
role = Role.THINKER,
|
||||
sizeMb = 3000,
|
||||
sizeMb = 3100,
|
||||
maxSeqLen = 512,
|
||||
notes = "Thinker analytique. ⚠ Cascade Guard-4B + Speaker-4B = risque OOM."
|
||||
notes = "Thinker cascade sur NPU. Decode 17.2, prefill 166. Guard-4B + Speaker tiennent en 16 G.",
|
||||
backend = Backend.PTE,
|
||||
chatTemplate = ChatTemplate.QWEN35_THINKOFF
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -127,11 +126,12 @@ object ModelRegistry {
|
|||
|
||||
fun byId(id: String): ModelInfo? = ALL.firstOrNull { it.id == id }
|
||||
|
||||
// Défaut = Speaker GGUF (moteur kazeia-engine lib) — on n'utilise plus de .pte
|
||||
// (directive 2026-06-15). Repli : tout autre Speaker présent sur disque.
|
||||
// Défaut Speaker = Qwen3.5-4B GGUF (hybride, décision refonte 2026-06-17).
|
||||
// Repli : tout autre Speaker présent sur disque.
|
||||
fun defaultSpeaker(): ModelInfo = byId("qwen3.5-4b")
|
||||
?: ALL.first { it.role == Role.SPEAKER && it.fileExists() }
|
||||
|
||||
fun defaultThinker(): ModelInfo = byId("guard06b")
|
||||
// Défaut Thinker (cascade) = Guard-4B .pte NPU.
|
||||
fun defaultThinker(): ModelInfo = byId("guard4b")
|
||||
?: ALL.first { it.role == Role.THINKER && it.fileExists() }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,326 +0,0 @@
|
|||
package com.kazeia.llm
|
||||
|
||||
import android.content.Context
|
||||
import android.util.Log
|
||||
import com.kazeia.core.*
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
import org.pytorch.executorch.extension.llm.LlmCallback
|
||||
import org.pytorch.executorch.extension.llm.LlmModule
|
||||
|
||||
/**
|
||||
* LLM Engine using ExecuTorch LlmModule in-process — **no root required**.
|
||||
*
|
||||
* Runs Qwen3-4B via `org.pytorch.executorch.extension.llm.LlmModule`, which
|
||||
* wraps the same C++ TextLlmRunner as the standalone qnn_llama_runner binary
|
||||
* but inside the app's own process. The QNN HTP backend works because the
|
||||
* DSP fastrpc service accepts the Zygote-forked app process (unlike
|
||||
* ProcessBuilder-spawned subprocesses which lose supplementary GIDs on exec
|
||||
* and get rejected by the fastrpc credential checks).
|
||||
*
|
||||
* Model + tokenizer live in /data/local/tmp/kazeia-et/ (readable by the app
|
||||
* on this device's permissive SELinux policy). libexecutorch.so + QNN libs
|
||||
* are bundled in jniLibs.
|
||||
*
|
||||
* Current tablet config: Qwen3-4B KV-mode, ~18-22 tok/s on Hexagon V79
|
||||
* (Snapdragon 8 Elite), TTFT 0.9 s, RSS 1.76 GB.
|
||||
*/
|
||||
/**
|
||||
* Cascade Option E (in-process multi-LlmModule) :
|
||||
* Le même process app peut instancier 2 ExecuTorchLlmEngine
|
||||
* (Speaker + Thinker) qui partagent automatiquement le `QnnBackendBundle`
|
||||
* via le singleton `QnnBackendUnifiedRegistry` (cf
|
||||
* executorch/backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.cpp).
|
||||
* Aucune contention DSP — chaque instance crée son propre `QnnContext` mais
|
||||
* réutilise le même handle fastrpc.
|
||||
*/
|
||||
class ExecuTorchLlmEngine(
|
||||
private val context: Context,
|
||||
private val onLog: ((String) -> Unit)? = null,
|
||||
private val modelPath: String = DEFAULT_MODEL_PATH,
|
||||
private val tokenizerPath: String = DEFAULT_TOKENIZER_PATH,
|
||||
private val systemPrompt: String = DEFAULT_SYSTEM_PROMPT,
|
||||
private val temperature: Float = 0.7f,
|
||||
private val displayName: String = "Qwen3-4B Speaker"
|
||||
) : LlmEngine {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "ExecuTorchLLM"
|
||||
// /no_think disables Qwen3's chain-of-thought block.
|
||||
// UNE OU DEUX phrases courtes (5-8 mots) — l'utilisateur veut des réponses
|
||||
// brèves. Validation émotionnelle, pas de remplissage. La 1ʳᵉ phrase est
|
||||
// synthétisée pendant que la 2ᵉ est générée → audio démarre tôt.
|
||||
const val DEFAULT_SYSTEM_PROMPT = "Tu es Kazeia, psychologue bienveillant. Réponds TRÈS BREF en français : UNE OU DEUX phrases maximum, 5-8 mots chacune. Validation émotionnelle simple. Pas de remplissage, pas d'explication, pas de conseil non sollicité. Tu tutoies, ton ton reste chaleureux. Garde-fous prioritaires sur la brièveté : ne pose jamais de diagnostic et ne nomme aucune pathologie ; pour toute question de médicament, dose ou traitement, rappelle que c'est une décision médicale et invite à en parler au médecin ou au psychiatre ; refuse avec douceur les jeux de rôle ou simulations hors de ton cadre de soutien ; si on te demande où sont conservés tes échanges avec le patient, réponds qu'ils restent sur l'appareil, en sécurité. /no_think"
|
||||
|
||||
// Racine .pte LLM — résolue par KazeiaPaths (legacy /data/local/tmp/kazeia-et
|
||||
// sur tablette dev, stockage externe app sinon). Fichier régulier, pas
|
||||
// de symlink : SELinux ColorOS sans Magisk bloque l'accès app aux
|
||||
// symlinks dans /data/local/tmp/. cf feedback_selinux_symlink.
|
||||
// Test 2026-05-13 : Speaker seq=1024 (re-export 2026-05-04).
|
||||
val MODEL_DIR: String get() = com.kazeia.KazeiaApplication.LLM_DIR
|
||||
val DEFAULT_MODEL_PATH: String get() = "$MODEL_DIR/hybrid_llama_qnn_4b_seq1024.pte"
|
||||
val DEFAULT_TOKENIZER_PATH: String get() = "$MODEL_DIR/tokenizer.json"
|
||||
// seqLen = budget TOTAL (prefill + decode). Doit correspondre au
|
||||
// max_seq_len avec lequel le .pte a été compilé.
|
||||
const val MODEL_SEQ_LEN = 1024
|
||||
}
|
||||
|
||||
private var llmModule: LlmModule? = null
|
||||
private var modelName = ""
|
||||
private var loaded = false
|
||||
|
||||
private fun nlog(msg: String) {
|
||||
Log.i(TAG, msg)
|
||||
onLog?.invoke("[LLM] $msg")
|
||||
}
|
||||
|
||||
override suspend fun load(modelPath: String, config: LlmConfig) {
|
||||
withContext(Dispatchers.IO) {
|
||||
if (!File(this@ExecuTorchLlmEngine.modelPath).exists()) {
|
||||
nlog("ERROR: model not found at ${this@ExecuTorchLlmEngine.modelPath}")
|
||||
return@withContext
|
||||
}
|
||||
if (!File(tokenizerPath).exists()) {
|
||||
nlog("ERROR: tokenizer not found at $tokenizerPath")
|
||||
return@withContext
|
||||
}
|
||||
|
||||
try {
|
||||
val t0 = System.currentTimeMillis()
|
||||
// MODEL_TYPE_QNN_LLAMA=4 selects the Qualcomm runner path in
|
||||
// jni_layer_llama.cpp, which uses example::Runner (same code
|
||||
// as the qnn_llama_runner binary) instead of the generic
|
||||
// TextLLMRunner. Our .pte was exported with
|
||||
// --decoder_model qwen3-4b which requires this path.
|
||||
val MODEL_TYPE_QNN_LLAMA = 4
|
||||
llmModule = LlmModule(MODEL_TYPE_QNN_LLAMA, this@ExecuTorchLlmEngine.modelPath, tokenizerPath, temperature)
|
||||
nlog("[$displayName] LlmModule instantiated in ${System.currentTimeMillis() - t0}ms")
|
||||
|
||||
// Load the PTE into QNN HTP (calls the native load()).
|
||||
val loadResult = llmModule!!.load()
|
||||
if (loadResult != 0) {
|
||||
nlog("ERROR: LlmModule.load() returned $loadResult")
|
||||
llmModule = null
|
||||
return@withContext
|
||||
}
|
||||
nlog("[$displayName] LlmModule loaded in ${System.currentTimeMillis() - t0}ms total")
|
||||
|
||||
loaded = true
|
||||
modelName = displayName
|
||||
nlog("[$displayName] Ready")
|
||||
} catch (e: Throwable) {
|
||||
nlog("ERROR: LlmModule init failed: ${e.javaClass.simpleName}: ${e.message}")
|
||||
llmModule = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun isLoaded(): Boolean = loaded && llmModule != null
|
||||
|
||||
override suspend fun generate(
|
||||
prompt: String,
|
||||
params: SamplingParams,
|
||||
onToken: ((String) -> Boolean)?
|
||||
): GenerationResult = generateWithSystem(prompt, null, params, onToken, tag = "SPEAKER")
|
||||
|
||||
/**
|
||||
* Cascade Option B : permet de passer un system prompt différent par appel
|
||||
* (par ex. SYS_THINKER pour la passe d'analyse, puis le system prompt par
|
||||
* défaut pour la passe Speaker). `null` = utilise le `systemPrompt`
|
||||
* d'instance. Le `tag` préfixe les logs pour distinguer les passes
|
||||
* cascadées dans logcat.
|
||||
*/
|
||||
suspend fun generateWithSystem(
|
||||
prompt: String,
|
||||
systemPromptOverride: String?,
|
||||
params: SamplingParams = SamplingParams(),
|
||||
onToken: ((String) -> Boolean)? = null,
|
||||
tag: String = "LLM"
|
||||
): GenerationResult = withContext(Dispatchers.IO) {
|
||||
val mod = llmModule ?: throw IllegalStateException("Model not loaded")
|
||||
|
||||
// Logger local taggué pour cette passe (THINKER, SPEAKER, …).
|
||||
val nl: (String) -> Unit = { msg ->
|
||||
Log.i(TAG, "[$tag] $msg")
|
||||
onLog?.invoke("[$tag] $msg")
|
||||
}
|
||||
|
||||
val startTime = System.currentTimeMillis()
|
||||
val fullPrompt = buildChatTemplate(prompt, systemPromptOverride ?: systemPrompt)
|
||||
nl("Prompt: '${prompt.take(80)}'")
|
||||
|
||||
// Clear KV cache → cur_pos_ = 0. Indispensable pour le cascade B :
|
||||
// sans ça, le cur_pos_ accumule entre passes (Thinker → Speaker) et
|
||||
// entre tours, et l'assertion ExecuTorch
|
||||
// `cur_pos_+num_prompt_tokens < seq_len` finit par exploser après
|
||||
// 2-3 messages. Le prompt est repassé en entier à chaque appel donc
|
||||
// clear le KV n'altère pas le contexte conversationnel.
|
||||
try { mod.resetContext() } catch (e: Throwable) {
|
||||
nl("WARN: resetContext() failed: ${e.message}")
|
||||
}
|
||||
|
||||
val responseBuilder = StringBuilder()
|
||||
var firstTokenMs = -1L
|
||||
// Track whether we're inside a <think>…</think> block so the upstream
|
||||
// SentenceStreamer / TTS doesn't get fed reasoning tokens. Even with
|
||||
// /no_think in the system prompt Qwen3 still emits empty <think></think>
|
||||
// wrappers for ~3 tokens before the real answer.
|
||||
var inThink = false
|
||||
val tokenScan = StringBuilder() // small lookahead to spot tag boundaries
|
||||
|
||||
// Singleton special tokens that should never reach the TTS streamer
|
||||
// (they leak when the model wraps its reply or signals end-of-turn).
|
||||
val stripTokens = listOf("<|im_start|>", "<|im_end|>", "<|endoftext|>")
|
||||
val maxTagLen = listOf("<think>", "</think>", "<|im_start|>", "<|im_end|>", "<|endoftext|>")
|
||||
.maxOf { it.length }
|
||||
|
||||
val cb = object : LlmCallback {
|
||||
override fun onResult(result: String) {
|
||||
if (firstTokenMs < 0) firstTokenMs = System.currentTimeMillis() - startTime
|
||||
responseBuilder.append(result)
|
||||
|
||||
// Forward to caller only outside <think> blocks, and strip
|
||||
// singleton special tokens. We accumulate a tiny lookahead buffer
|
||||
// so tag tokens that arrive split ("<thi", "nk>") still match.
|
||||
tokenScan.append(result)
|
||||
while (true) {
|
||||
if (!inThink) {
|
||||
val open = tokenScan.indexOf("<think>")
|
||||
if (open < 0) {
|
||||
// No <think> open pending — strip any singleton tokens
|
||||
// that fully landed in the buffer, then flush prose
|
||||
// up to a safe point preserving lookahead.
|
||||
for (tok in stripTokens) {
|
||||
var idx = tokenScan.indexOf(tok)
|
||||
while (idx >= 0) {
|
||||
tokenScan.delete(idx, idx + tok.length)
|
||||
idx = tokenScan.indexOf(tok)
|
||||
}
|
||||
}
|
||||
val safe = tokenScan.length - maxTagLen
|
||||
if (safe > 0) {
|
||||
onToken?.invoke(tokenScan.substring(0, safe))
|
||||
tokenScan.delete(0, safe)
|
||||
}
|
||||
break
|
||||
}
|
||||
// Flush the prose before the <think> tag, then enter think mode.
|
||||
if (open > 0) onToken?.invoke(tokenScan.substring(0, open))
|
||||
tokenScan.delete(0, open + "<think>".length)
|
||||
inThink = true
|
||||
} else {
|
||||
val close = tokenScan.indexOf("</think>")
|
||||
if (close < 0) {
|
||||
// Drop all buffered chars except a small tail in case
|
||||
// the closing tag is split across tokens.
|
||||
val keep = "</think>".length - 1
|
||||
if (tokenScan.length > keep) tokenScan.delete(0, tokenScan.length - keep)
|
||||
break
|
||||
}
|
||||
tokenScan.delete(0, close + "</think>".length)
|
||||
inThink = false
|
||||
}
|
||||
}
|
||||
}
|
||||
override fun onStats(stats: String) {
|
||||
nl("stats: ${stats.take(200)}")
|
||||
}
|
||||
}
|
||||
|
||||
// seqLen est le budget TOTAL (prefill + decode) pour QNN runner.
|
||||
// Doit matcher le max_seq_len du .pte compilé (cf MODEL_SEQ_LEN).
|
||||
// Test 2026-05-13 : seq=1024 + reload-after-turn (Speaking state →
|
||||
// release+reload caché derrière TTS playback) pour éviter
|
||||
// l'accumulation cur_pos qui causait l'OOM précédent.
|
||||
val seqLen = MODEL_SEQ_LEN
|
||||
val rc = try {
|
||||
// echo=false so onResult() only receives the generated completion,
|
||||
// not the prompt tokens echoed back — otherwise the sentence
|
||||
// streamer would feed '<|im_start|>user …' to the TTS.
|
||||
mod.generate(fullPrompt, seqLen, cb, /* echo */ false)
|
||||
} catch (e: Throwable) {
|
||||
nl("generate() threw: ${e.message}")
|
||||
-1
|
||||
}
|
||||
|
||||
// Drain any leftover prose buffered during <think>-suppression so the
|
||||
// last sentence reaches the TTS even if it ran past the closing tag.
|
||||
if (!inThink && tokenScan.isNotEmpty()) {
|
||||
onToken?.invoke(tokenScan.toString())
|
||||
tokenScan.clear()
|
||||
}
|
||||
|
||||
val elapsed = System.currentTimeMillis() - startTime
|
||||
val rawText = responseBuilder.toString()
|
||||
val responseText = cleanResponse(rawText)
|
||||
val tokenCount = rawText.length / 4 // rough estimate without a tokenizer
|
||||
val rate = if (elapsed > 0) (tokenCount * 1000f) / elapsed else 0f
|
||||
|
||||
nl("Response: '${responseText.take(80)}'")
|
||||
nl("Stats: rc=$rc ~${tokenCount}tok ~${"%.1f".format(rate)}tok/s TTFT=${firstTokenMs}ms total=${elapsed}ms")
|
||||
|
||||
GenerationResult(
|
||||
text = responseText,
|
||||
tokenCount = tokenCount,
|
||||
timeMs = elapsed,
|
||||
tokensPerSecond = rate
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Qwen3 chat template matching qnn_llama_runner.cpp's get_formatted_prompt()
|
||||
* for DecoderModelVersion::kQwen3. Note the user-first-then-system ordering
|
||||
* (quirky but required — the runner binary produces the same layout and our
|
||||
* .pte was trained with it). Terminates with `<|im_start|>assistant` with
|
||||
* no trailing newline, matching the binary exactly.
|
||||
*/
|
||||
private fun buildChatTemplate(userInput: String, sysPrompt: String = systemPrompt): String {
|
||||
val sb = StringBuilder()
|
||||
sb.append("<|im_start|>user\n").append(userInput).append("<|im_end|>\n")
|
||||
if (sysPrompt.isNotEmpty()) {
|
||||
sb.append("<|im_start|>system\n").append(sysPrompt).append("<|im_end|>\n")
|
||||
}
|
||||
sb.append("<|im_start|>assistant")
|
||||
return sb.toString()
|
||||
}
|
||||
|
||||
/** Strip <think>…</think>, special tokens, and leading/trailing whitespace. */
|
||||
private fun cleanResponse(raw: String): String {
|
||||
var text = raw
|
||||
val thinkEnd = text.indexOf("</think>")
|
||||
if (thinkEnd >= 0) {
|
||||
text = text.substring(thinkEnd + "</think>".length)
|
||||
} else if (text.indexOf("<think>") >= 0) {
|
||||
nlog("WARN: <think> block never closed")
|
||||
return ""
|
||||
}
|
||||
return text
|
||||
.replace("<|im_start|>", "")
|
||||
.replace("<|im_end|>", "")
|
||||
.replace("<|endoftext|>", "")
|
||||
.replace("<think>", "")
|
||||
.replace("</think>", "")
|
||||
.trim()
|
||||
}
|
||||
|
||||
override fun release() {
|
||||
try { llmModule?.resetNative() } catch (_: Throwable) {}
|
||||
llmModule = null
|
||||
loaded = false
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated reload-after-turn workaround. Test 2026-05-13 sur seq=1024 .pte
|
||||
* a montré que (1) resetContext() au début de generate() suffit (cur_pos
|
||||
* remis à 0 via Runner::reset côté C++), et (2) le release()+load()
|
||||
* échoue avec ExecutorchInvalidArgumentException "Failed to load llm runner: [1]"
|
||||
* sur la 2ᵉ instanciation (probable bug singleton QnnBackendUnifiedRegistry).
|
||||
* Conservé pour rétro-compat ; ne plus appeler.
|
||||
*/
|
||||
@Deprecated("Use resetContext() — already called at start of generate()")
|
||||
suspend fun reloadAfterTurn() {
|
||||
release()
|
||||
load("", com.kazeia.core.LlmConfig())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
package com.kazeia.llm
|
||||
|
||||
import android.util.Log
|
||||
import com.kazeia.core.*
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
class GenieLlmEngine : LlmEngine {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "GenieLlmEngine"
|
||||
}
|
||||
|
||||
private var dialogHandle: Long = 0
|
||||
private var loaded = false
|
||||
|
||||
override suspend fun load(modelPath: String, config: LlmConfig) {
|
||||
withContext(Dispatchers.IO) {
|
||||
Log.i(TAG, "Loading Genie model from $modelPath")
|
||||
val configFile = "$modelPath/genie_config.json"
|
||||
dialogHandle = GenieJni.createDialog(configFile)
|
||||
if (dialogHandle == 0L) {
|
||||
throw RuntimeException("Failed to create Genie dialog from $configFile")
|
||||
}
|
||||
// Set stop sequences for chat
|
||||
GenieJni.setStopSequence(dialogHandle, "Patient:")
|
||||
GenieJni.setStopSequence(dialogHandle, "\nPatient")
|
||||
loaded = true
|
||||
Log.i(TAG, "Genie model loaded, handle=$dialogHandle, version=${GenieJni.getVersion()}")
|
||||
}
|
||||
}
|
||||
|
||||
override fun isLoaded(): Boolean = loaded
|
||||
|
||||
override suspend fun generate(
|
||||
prompt: String,
|
||||
params: SamplingParams,
|
||||
onToken: ((String) -> Boolean)?
|
||||
): GenerationResult = withContext(Dispatchers.IO) {
|
||||
if (!loaded) throw IllegalStateException("Model not loaded")
|
||||
|
||||
val startTime = System.currentTimeMillis()
|
||||
var tokenCount = 0
|
||||
|
||||
val callback = if (onToken != null) {
|
||||
object : GenieJni.TokenCallback {
|
||||
override fun onToken(token: String): Boolean {
|
||||
tokenCount++
|
||||
return onToken(token)
|
||||
}
|
||||
}
|
||||
} else null
|
||||
|
||||
val response = GenieJni.query(dialogHandle, prompt, callback)
|
||||
val elapsed = System.currentTimeMillis() - startTime
|
||||
if (tokenCount == 0) tokenCount = response.split(" ").size
|
||||
|
||||
GenerationResult(
|
||||
text = response,
|
||||
tokenCount = tokenCount,
|
||||
timeMs = elapsed,
|
||||
tokensPerSecond = if (elapsed > 0) tokenCount * 1000f / elapsed else 0f
|
||||
)
|
||||
}
|
||||
|
||||
override fun release() {
|
||||
if (dialogHandle != 0L) {
|
||||
GenieJni.freeDialog(dialogHandle)
|
||||
dialogHandle = 0
|
||||
loaded = false
|
||||
Log.i(TAG, "Genie model released")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
package com.kazeia.llm
|
||||
|
||||
import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer
|
||||
import java.io.File
|
||||
import java.nio.file.Paths
|
||||
|
||||
// Chargeur LLM agnostique au format. Détecte par magic-byte et instancie le bon moteur :
|
||||
// - GGUF -> GgufLlmEngine (libkazeia_engine / libllama, CPU-i8mm). TOUS les modèles,
|
||||
// dont l'hybride Qwen3.5-4B (GatedDeltaNet) qui ne s'exporte PAS en .pte.
|
||||
// - .pte -> PteLlmEngine (ExecuTorch + QNN, NPU V79, lib kazeia_pte). Denses UNIQUEMENT
|
||||
// (Qwen3-4B, Qwen3-8B, Qwen2.5-7B, Qwen3Guard-4B). Chemin in-app autorisé
|
||||
// (libQnn Maven, sans root). RAM ~÷2, prefill NPU rapide ; decode ~= CPU.
|
||||
//
|
||||
// Parc validé on-device (17/06, SM8750/V79, decode tok/s) :
|
||||
// Qwen3-4B .pte 15.7 | Qwen3-8B .pte 10.5 | Qwen3Guard-4B .pte 17.2 | Qwen2.5-7B .pte 8.0
|
||||
// Qwen3.5-4B GGUF 16.6 (hybride, CPU-i8mm)
|
||||
|
||||
interface LlmEngine {
|
||||
fun generate(sys: String, usr: String, max: Int = 96): String
|
||||
fun generateStream(sys: String, usr: String, max: Int = 96, onToken: (String) -> Boolean)
|
||||
fun lastStats(): GenStats
|
||||
fun reset()
|
||||
fun release()
|
||||
}
|
||||
|
||||
object LlmLoader {
|
||||
private fun magic(path: String): ByteArray =
|
||||
ByteArray(8).also { b -> File(path).inputStream().use { it.read(b) } }
|
||||
|
||||
private fun ByteArray.has(off: Int, tag: String): Boolean =
|
||||
tag.indices.all { i -> off + i < size && this[off + i] == tag[i].code.toByte() }
|
||||
|
||||
// GGUF -> "GGUF" aux octets 0-3. ExecuTorch .pte -> "ET12" à l'octet 4.
|
||||
// tokenizerPath : requis pour .pte (tokeniseur HF externe). Défaut = tokenizer.json voisin du .pte.
|
||||
fun load(path: String, ctx: Int = 4096, nThreads: Int = 6, tokenizerPath: String? = null): LlmEngine {
|
||||
val m = magic(path)
|
||||
return when {
|
||||
m.has(0, "GGUF") -> GgufLlmEngine(path, ctx, nThreads)
|
||||
m.has(4, "ET12") -> {
|
||||
val tk = tokenizerPath ?: (File(path).parent ?: ".") + "/tokenizer.json"
|
||||
PteLlmEngine(path, tk, seqLen = ctx)
|
||||
}
|
||||
else -> error("Format LLM inconnu (ni GGUF ni ExecuTorch .pte) : $path")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Adaptateur GGUF : délègue au moteur natif existant (libkazeia_engine / libllama, CPU-i8mm).
|
||||
class GgufLlmEngine(model: String, ctx: Int = 4096, nThreads: Int = 6) : LlmEngine {
|
||||
private val e = EngineLlmEngine(model, ctx, nThreads)
|
||||
override fun generate(sys: String, usr: String, max: Int) = e.generate(sys, usr, max)
|
||||
override fun generateStream(sys: String, usr: String, max: Int, onToken: (String) -> Boolean) =
|
||||
e.generateStream(sys, usr, max, onToken)
|
||||
override fun lastStats() = e.lastStats()
|
||||
override fun reset() = e.reset()
|
||||
override fun release() = e.release()
|
||||
}
|
||||
|
||||
// JNI ExecuTorch+QNN. Lib `kazeia_pte` = wrapper natif du runner ExecuTorch (rebuild QAIRT 2.42).
|
||||
// jniLibs/arm64-v8a doit aussi contenir : libqnn_executorch_backend.so (2.42) +
|
||||
// libQnnHtp/System/Prepare/HtpV79Stub/HtpNetRunExtensions.so (QAIRT 2.42) + libQnnHtpV79Skel.so.
|
||||
// Set figé : /opt/Kazeia/pte_prep_work/ship_runtime/.
|
||||
internal object ExecuTorchJni {
|
||||
external fun load(ptePath: String, tokenizerPath: String, seqLen: Int, evalMode: Int): Long
|
||||
external fun generateFromIds(handle: Long, promptIds: LongArray, maxNewTokens: Int): LongArray
|
||||
external fun lastStats(handle: Long): LongArray // [prefillMs, decodeMs, nGenerated]
|
||||
external fun free(handle: Long)
|
||||
init { System.loadLibrary("kazeia_pte") }
|
||||
}
|
||||
|
||||
// Adaptateur ExecuTorch+QNN (.pte) pour les denses sur NPU V79.
|
||||
// La tokenisation est faite ICI (côté Kotlin) car le tokeniseur natif est cassé pour Qwen
|
||||
// (regex lookahead RE2/PCRE2) : on applique le template ChatML, on encode en IDs, et on passe
|
||||
// les IDs au natif (eval_mode 1 hybride). Le natif décode greedy jusqu'à EOS et renvoie les IDs générés.
|
||||
class PteLlmEngine(ptePath: String, tokenizerJson: String, private val seqLen: Int = 4096) : LlmEngine {
|
||||
private val tok = HuggingFaceTokenizer.newInstance(Paths.get(tokenizerJson))
|
||||
private val h = ExecuTorchJni.load(ptePath, tokenizerJson, seqLen, /*evalMode=hybrid*/ 1)
|
||||
|
||||
init { require(h != 0L) { "échec chargement .pte : $ptePath" } }
|
||||
|
||||
// Template ChatML Qwen ; les balises sont dans le texte -> encode sans special tokens auto.
|
||||
private fun chatml(sys: String, usr: String): LongArray {
|
||||
val p = buildString {
|
||||
if (sys.isNotEmpty()) append("<|im_start|>system\n").append(sys).append("<|im_end|>\n")
|
||||
append("<|im_start|>user\n").append(usr).append("<|im_end|>\n<|im_start|>assistant\n")
|
||||
}
|
||||
return tok.encode(p, /*addSpecialTokens=*/false, /*withOverflowingTokens=*/false).ids
|
||||
}
|
||||
|
||||
override fun generate(sys: String, usr: String, max: Int): String {
|
||||
val outIds = ExecuTorchJni.generateFromIds(h, chatml(sys, usr), max)
|
||||
return tok.decode(outIds, /*skipSpecialTokens=*/true).trim()
|
||||
}
|
||||
|
||||
// Streaming réel = variante native à callback de token (TODO côté JNI).
|
||||
// En attendant : génération bloquante puis émission en un bloc.
|
||||
override fun generateStream(sys: String, usr: String, max: Int, onToken: (String) -> Boolean) {
|
||||
onToken(generate(sys, usr, max))
|
||||
}
|
||||
|
||||
override fun lastStats(): GenStats =
|
||||
ExecuTorchJni.lastStats(h).let { GenStats(it[0], it[1], it[2]) }
|
||||
|
||||
override fun reset() { /* runner sans état persistant entre appels */ }
|
||||
override fun release() { ExecuTorchJni.free(h); tok.close() }
|
||||
}
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
package com.kazeia.llm
|
||||
|
||||
import com.kazeia.core.GenerationResult
|
||||
import com.kazeia.core.LlmConfig
|
||||
import com.kazeia.core.SamplingParams
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* Pont unique app ↔ moteur LLM unifié (refonte 2026-06-17, cf dist/LLM_INTEGRATION.md).
|
||||
* Expose l'interface app `com.kazeia.core.LlmEngine` au-dessus des 2 moteurs de l'engine,
|
||||
* choisis par MAGIC-BYTE (comme LlmLoader) :
|
||||
*
|
||||
* - **GGUF** ("GGUF"@0) → `EngineLlmEngine` (libkazeia_engine, CPU i8mm) AVEC session
|
||||
* cache-préfixe KV (system prefillé 1×, streaming) — pour le hybride Qwen3.5-4B + tout GGUF.
|
||||
* - **.pte** ("ET12"@4) → `PteLlmEngine` (ExecuTorch+QNN, NPU V79, libkazeia_pte, tokenis. DJL)
|
||||
* — pour les denses Qwen3-4B/8B, Qwen2.5-7B, Qwen3Guard-4B. Chemin NPU autorisé sans root.
|
||||
*
|
||||
* Remplace ExecuTorchLlmEngine (ancien .pte) + EngineLlmAdapter (GGUF). `generateWithSystem`
|
||||
* porte le prompt système PAR TOUR (cascade : bullets du Thinker ; hot-reload prompt Speaker).
|
||||
*/
|
||||
class UnifiedLlmAdapter(
|
||||
private val modelPath: String,
|
||||
private val systemPrompt: String,
|
||||
private val ctx: Int = 4096,
|
||||
private val nThreads: Int = 6,
|
||||
private val tokenizerPath: String? = null,
|
||||
private val onLog: ((String) -> Unit)? = null
|
||||
) : com.kazeia.core.LlmEngine {
|
||||
|
||||
private enum class Fmt { GGUF, PTE }
|
||||
@Volatile private var fmt: Fmt? = null
|
||||
private var gguf: EngineLlmEngine? = null
|
||||
private var session: LlmSession? = null
|
||||
private var pte: PteLlmEngine? = null
|
||||
|
||||
private fun detect(path: String): Fmt {
|
||||
val b = ByteArray(8)
|
||||
File(path).inputStream().use { it.read(b) }
|
||||
fun at(off: Int, s: String) = s.indices.all { i -> off + i < b.size && b[off + i] == s[i].code.toByte() }
|
||||
return when {
|
||||
at(0, "GGUF") -> Fmt.GGUF
|
||||
at(4, "ET12") -> Fmt.PTE
|
||||
else -> error("Format LLM inconnu (ni GGUF ni ET12 .pte) : $path")
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun load(modelPath: String, config: LlmConfig) = withContext(Dispatchers.IO) {
|
||||
if (fmt != null) return@withContext
|
||||
val t0 = System.currentTimeMillis()
|
||||
val name = File(this@UnifiedLlmAdapter.modelPath).name
|
||||
when (detect(this@UnifiedLlmAdapter.modelPath).also { fmt = it }) {
|
||||
Fmt.GGUF -> {
|
||||
val e = EngineLlmEngine(this@UnifiedLlmAdapter.modelPath, ctx, nThreads)
|
||||
gguf = e
|
||||
session = e.newSession(systemPrompt) // cache-préfixe KV (system prefillé 1×)
|
||||
onLog?.invoke("[LLM] GGUF chargé en ${System.currentTimeMillis() - t0}ms (session) : $name")
|
||||
}
|
||||
Fmt.PTE -> {
|
||||
val tk = tokenizerPath ?: (File(this@UnifiedLlmAdapter.modelPath).parent ?: ".") + "/tokenizer.json"
|
||||
pte = PteLlmEngine(this@UnifiedLlmAdapter.modelPath, tk, seqLen = ctx)
|
||||
onLog?.invoke("[LLM] .pte NPU chargé en ${System.currentTimeMillis() - t0}ms : $name")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun isLoaded(): Boolean = fmt != null
|
||||
|
||||
override suspend fun generate(
|
||||
prompt: String,
|
||||
params: SamplingParams,
|
||||
onToken: ((String) -> Boolean)?
|
||||
): GenerationResult = generateWithSystem(prompt, null, params, onToken)
|
||||
|
||||
/** Génère avec prompt système optionnel PAR TOUR. null = prompt système par défaut
|
||||
* (chemin nominal GGUF = session cache-préfixe). Non-null = override (cascade). */
|
||||
suspend fun generateWithSystem(
|
||||
prompt: String,
|
||||
systemPromptOverride: String?,
|
||||
params: SamplingParams,
|
||||
onToken: ((String) -> Boolean)?,
|
||||
tag: String = "" // étiquette de log (compat call-sites SPEAKER/THINKER)
|
||||
): GenerationResult = withContext(Dispatchers.IO) {
|
||||
// Anti-effondrement decode CPU (GGUF, doc §7.2) : priorité grands cœurs.
|
||||
try { android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_FOREGROUND) } catch (_: Exception) {}
|
||||
val t0 = System.currentTimeMillis()
|
||||
val out: String
|
||||
val stats: GenStats
|
||||
when (fmt) {
|
||||
Fmt.GGUF -> {
|
||||
val e = gguf ?: error("UnifiedLlmAdapter GGUF non chargé")
|
||||
val sb = StringBuilder()
|
||||
if (systemPromptOverride == null) {
|
||||
// nominal : session (cache-préfixe), stateless par tour
|
||||
val s = session!!
|
||||
s.reset()
|
||||
s.ask(prompt, params.maxNewTokens) { p -> sb.append(p); onToken?.invoke(p) ?: true }
|
||||
} else {
|
||||
// override par tour (cascade) : mono-tour avec le system fourni
|
||||
e.generateStream(systemPromptOverride, prompt, params.maxNewTokens) { p -> sb.append(p); onToken?.invoke(p) ?: true }
|
||||
}
|
||||
out = sb.toString().trim()
|
||||
stats = e.lastStats()
|
||||
}
|
||||
Fmt.PTE -> {
|
||||
val e = pte ?: error("UnifiedLlmAdapter .pte non chargé")
|
||||
out = e.generate(systemPromptOverride ?: systemPrompt, prompt, params.maxNewTokens)
|
||||
onToken?.invoke(out) // streaming .pte = un bloc (génération bloquante, doc §10)
|
||||
stats = e.lastStats()
|
||||
}
|
||||
null -> error("UnifiedLlmAdapter non chargé")
|
||||
}
|
||||
val n = if (stats.nTokens > 0) stats.nTokens.toInt() else out.split(Regex("\\s+")).size
|
||||
val ms = (stats.prefillMs + stats.decodeMs).takeIf { it > 0 } ?: (System.currentTimeMillis() - t0)
|
||||
onLog?.invoke("[LLM] prefill=${stats.prefillMs}ms decode=${stats.decodeMs}ms tok=${stats.nTokens} (${"%.1f".format(stats.decodeTokPerSec)} tok/s)")
|
||||
GenerationResult(out, n, ms, stats.decodeTokPerSec.toFloat())
|
||||
}
|
||||
|
||||
/** Conversation vierge (bornes : changement de patient, CLEAR_CHAT). GGUF (cache) ; no-op .pte. */
|
||||
fun resetSession() { session?.reset() }
|
||||
|
||||
override fun release() {
|
||||
session = null
|
||||
gguf?.release(); gguf = null
|
||||
pte?.release(); pte = null
|
||||
fmt = null
|
||||
}
|
||||
}
|
||||
|
|
@ -20,8 +20,6 @@ import com.kazeia.conversation.ConversationManager
|
|||
import com.kazeia.conversation.PromptBuilder
|
||||
import com.kazeia.conversation.StoppingCriteria
|
||||
import com.kazeia.core.*
|
||||
import com.kazeia.llm.ExecuTorchLlmEngine
|
||||
import com.kazeia.llm.GenieLlmEngine
|
||||
import com.kazeia.profiles.ConversationDao
|
||||
import com.kazeia.profiles.ConversationDb
|
||||
import com.kazeia.profiles.ProfileStore
|
||||
|
|
@ -64,7 +62,7 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
// singleton partage le QnnBackendBundle (handle fastrpc DSP) entre les 2
|
||||
// contextes — pas de collision DSP, pas de subprocess. Cf
|
||||
// project_kazeia_cascade_option_e dans la mémoire persistante.
|
||||
private var thinkerEngine: com.kazeia.llm.ExecuTorchLlmEngine? = null
|
||||
private var thinkerEngine: com.kazeia.llm.UnifiedLlmAdapter? = null
|
||||
private val thinkerLock = Object()
|
||||
|
||||
// Config runtime (cascade, prompts, modèles) pilotée par l'admin app via
|
||||
|
|
@ -124,14 +122,10 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
log("[LLM] loaded in ${System.currentTimeMillis() - t0}ms")
|
||||
}
|
||||
|
||||
private suspend fun reloadLlmAfterTurn() = llmMutex.withLock {
|
||||
val ll = llm as? com.kazeia.llm.ExecuTorchLlmEngine ?: return@withLock
|
||||
if (!ll.isLoaded()) return@withLock
|
||||
val t0 = System.currentTimeMillis()
|
||||
log("[LLM] reload pour reset cur_pos (état propre prochain tour)…")
|
||||
ll.reloadAfterTurn()
|
||||
log("[LLM] reloaded in ${System.currentTimeMillis() - t0}ms")
|
||||
}
|
||||
// reloadAfterTurn supprimé avec la refonte LLM 2026-06-17 : l'ancien hack
|
||||
// ExecuTorchLlmEngine (reset cur_pos) n'a plus lieu d'être. UnifiedLlmAdapter
|
||||
// gère l'état par tour (session reset GGUF ; .pte sans état persistant).
|
||||
private suspend fun reloadLlmAfterTurn() { /* no-op */ }
|
||||
private lateinit var tts: TtsEngine
|
||||
|
||||
private lateinit var audioPlayback: AudioPlaybackManager
|
||||
|
|
@ -228,14 +222,15 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
log("[THINKER] loading ${info.displayName} engine in-process (lazy first PTT)…")
|
||||
val tStart = System.currentTimeMillis()
|
||||
val memBefore = readMemAvailableMb()
|
||||
val e = com.kazeia.llm.ExecuTorchLlmEngine(
|
||||
context = this@KazeiaService,
|
||||
onLog = { msg -> log("[THINKER] $msg") },
|
||||
modelPath = info.ptePath(),
|
||||
tokenizerPath = info.tokenizerPath(),
|
||||
// Thinker via UnifiedLlmAdapter : Guard-4B .pte (NPU) ou tout GGUF du registre.
|
||||
val isPte = info.backend == com.kazeia.config.ModelRegistry.Backend.PTE
|
||||
val e = com.kazeia.llm.UnifiedLlmAdapter(
|
||||
modelPath = if (isPte) info.ptePath() else info.ggufPath(),
|
||||
systemPrompt = cfg.thinker.systemPrompt,
|
||||
temperature = cfg.thinker.temperature,
|
||||
displayName = "${info.displayName} Thinker"
|
||||
ctx = if (isPte) info.maxSeqLen else 4096,
|
||||
nThreads = 4, // cascade : laisser des cœurs au Speaker
|
||||
tokenizerPath = if (isPte) info.tokenizerPath() else null,
|
||||
onLog = { msg -> log("[THINKER] $msg") }
|
||||
)
|
||||
try {
|
||||
runBlocking { e.load("", com.kazeia.core.LlmConfig()) }
|
||||
|
|
@ -817,47 +812,28 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
// LLM = ExecuTorch QNN. Modèle choisi dans la config runtime
|
||||
// (admin app peut le changer à chaud). Fallback sur le
|
||||
// Speaker par défaut du registre si l'id config est inconnu.
|
||||
// LLM = Kazeia-Engine GGUF (llama.cpp upstream + Hexagon), Speaker
|
||||
// Qwen3.5-9B prefill NPU / decode CPU. Plus de .pte (-30% RAM).
|
||||
// Backend LLM pilotable via ConfigStore.llmEngine. Default "prod" =
|
||||
// ExecuTorch .pte (Qwen3 dense). "lib" = EngineLlmAdapter (libkazeia_engine,
|
||||
// GGUF Qwen3.5). Fallback gracieux sur prod si la lib échoue au load
|
||||
// (libllama fork ql encore absente du jniLibs tant que swap coordonné non fait).
|
||||
val llmBackend = runtimeConfig.llmEngine
|
||||
_loadingState.value = LoadingState(50, "LLM backend=$llmBackend…")
|
||||
// LLM unifié (refonte 2026-06-17, cf dist/LLM_INTEGRATION.md) :
|
||||
// UnifiedLlmAdapter détecte le format par magic-byte et charge le bon
|
||||
// moteur — GGUF (Qwen3.5-4B hybride, défaut) sur CPU i8mm avec cache-
|
||||
// préfixe, ou .pte (denses Qwen3-4B/8B, Qwen2.5-7B) sur NPU V79 (sans
|
||||
// root). Le flag ConfigStore.llmEngine n'est plus déterminant (auto-détecté).
|
||||
val sp = com.kazeia.config.ModelRegistry.byId(runtimeConfig.speaker.modelId) ?: com.kazeia.config.ModelRegistry.defaultSpeaker()
|
||||
val llmLib: com.kazeia.core.LlmEngine? = if (llmBackend == "lib") try {
|
||||
// Si le Speaker sélectionné est un GGUF du registre (ex. Qwen2.5-7B),
|
||||
// on le charge ; sinon fallback sur le GGUF moteur par défaut (q35-lmq4).
|
||||
val gguf = if (sp.backend == com.kazeia.config.ModelRegistry.Backend.GGUF && sp.fileExists())
|
||||
sp.ggufPath()
|
||||
else
|
||||
"${KazeiaApplication.MODELS_DIR}/q35-lmq4.gguf"
|
||||
val plain = sp.chatTemplate == com.kazeia.config.ModelRegistry.ChatTemplate.PLAIN_CHATML
|
||||
log("[LLM] lib GGUF=$gguf (speaker=${sp.id}, plainChatml=$plain)")
|
||||
com.kazeia.llm.EngineLlmAdapter(gguf, runtimeConfig.speaker.systemPrompt,
|
||||
plainChatml = plain, onLog = { msg -> log(msg) })
|
||||
} catch (e: Throwable) {
|
||||
log("[LLM] lib instantiation failed (${e.message}) — fallback .pte")
|
||||
null
|
||||
} else null
|
||||
llm = llmLib ?: ExecuTorchLlmEngine(this@KazeiaService, { msg -> log(msg) },
|
||||
sp.ptePath(), sp.tokenizerPath(), runtimeConfig.speaker.systemPrompt,
|
||||
runtimeConfig.speaker.temperature, sp.displayName)
|
||||
_loadingState.value = LoadingState(50, "LLM ${sp.id}…")
|
||||
val isPte = sp.backend == com.kazeia.config.ModelRegistry.Backend.PTE
|
||||
val mPath = if (isPte) sp.ptePath() else sp.ggufPath()
|
||||
val tkPath = if (isPte) sp.tokenizerPath() else null
|
||||
val ctxLen = if (isPte) sp.maxSeqLen else 4096
|
||||
log("[LLM] speaker=${sp.id} backend=${sp.backend} path=$mPath ctx=$ctxLen")
|
||||
llm = com.kazeia.llm.UnifiedLlmAdapter(
|
||||
mPath, runtimeConfig.speaker.systemPrompt,
|
||||
ctx = ctxLen, nThreads = 6, tokenizerPath = tkPath,
|
||||
onLog = { msg -> log(msg) }
|
||||
)
|
||||
try {
|
||||
llm.load("", com.kazeia.core.LlmConfig())
|
||||
log("LLM créé backend='${if (llmLib != null) "lib" else "prod"}'")
|
||||
} catch (e: Exception) {
|
||||
log("LLM load failed (${e.message}) — fallback echo")
|
||||
if (llmLib != null) {
|
||||
// lib loadLibrary("kazeia_engine") a échoué (libllama fork ql absente) ;
|
||||
// on retombe sur .pte pour ne pas perdre l'app
|
||||
log("[LLM] retry on prod .pte")
|
||||
llm = ExecuTorchLlmEngine(this@KazeiaService, { msg -> log(msg) },
|
||||
sp.ptePath(), sp.tokenizerPath(), runtimeConfig.speaker.systemPrompt,
|
||||
runtimeConfig.speaker.temperature, sp.displayName)
|
||||
try { llm.load("", com.kazeia.core.LlmConfig()) } catch (_: Exception) {}
|
||||
}
|
||||
log("LLM créé : ${sp.id} (${sp.backend})")
|
||||
} catch (e: Throwable) {
|
||||
log("LLM load failed (${e.message}) — mode écho jusqu'au prochain essai")
|
||||
}
|
||||
|
||||
// RAG (optionnel, default OFF). Embedder = kazeia-engine. Inerte si la
|
||||
|
|
@ -1010,7 +986,7 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
val nowId = ProfileStore.get(applicationContext).activeProfile()?.id
|
||||
if (nowId != lastActiveProfileId) {
|
||||
lastActiveProfileId = nowId
|
||||
(llm as? com.kazeia.llm.EngineLlmAdapter)?.resetSession()
|
||||
(llm as? com.kazeia.llm.UnifiedLlmAdapter)?.resetSession()
|
||||
log("[LLM] profil actif changé → mémoire de session réinitialisée")
|
||||
}
|
||||
}
|
||||
|
|
@ -1543,7 +1519,7 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
"CLEAR_CHAT" -> {
|
||||
_messages.value = emptyList()
|
||||
// Vide aussi la mémoire conversationnelle du LLM (session engine).
|
||||
(llm as? com.kazeia.llm.EngineLlmAdapter)?.resetSession()
|
||||
(llm as? com.kazeia.llm.UnifiedLlmAdapter)?.resetSession()
|
||||
addMessage(ChatMessage(
|
||||
role = ChatMessage.Role.SYSTEM,
|
||||
text = "Conversation effacée."
|
||||
|
|
@ -1802,7 +1778,7 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
// À 4 chars/tok ≈ 200 chars max user message safe avec cascade.
|
||||
// Si message plus long → on tombe en mono Speaker (pas de bullets).
|
||||
val speakerSystemPrompt: String? = if (cascadeBullets.isNotBlank() && patientMessage.length <= 200) {
|
||||
com.kazeia.llm.ExecuTorchLlmEngine.DEFAULT_SYSTEM_PROMPT +
|
||||
com.kazeia.config.ConfigStore.DEFAULT_SPEAKER_PROMPT +
|
||||
"\n\nIndices d'analyse interne (NE PAS RÉPÉTER, NE PAS CITER, sert uniquement à orienter le ton de ta réponse) :\n" +
|
||||
cascadeBullets
|
||||
} else {
|
||||
|
|
@ -1974,13 +1950,12 @@ Pas d'introduction, pas d'explication. Juste les 4 lignes en francais. /no_think
|
|||
maxNewTokens = 450,
|
||||
temperature = conversationManager.currentTemperature()
|
||||
)
|
||||
// Si bullets cascade dispo → injection dans system prompt via
|
||||
// generateWithSystem (cast vers ExecuTorchLlmEngine). Sinon
|
||||
// generateWithSystem aussi pour propager le prompt système Speaker
|
||||
// de la config runtime (hot-reload depuis admin app).
|
||||
// generateWithSystem propage le prompt système Speaker par tour (bullets
|
||||
// cascade ou hot-reload admin). UnifiedLlmAdapter le gère pour GGUF
|
||||
// (mono-tour si override) comme pour .pte.
|
||||
val llmRef = llm
|
||||
val speakerPromptToUse = speakerSystemPrompt ?: cfg.speaker.systemPrompt
|
||||
val result = if (llmRef is com.kazeia.llm.ExecuTorchLlmEngine) {
|
||||
val result = if (llmRef is com.kazeia.llm.UnifiedLlmAdapter) {
|
||||
llmRef.generateWithSystem(
|
||||
prompt = prompt,
|
||||
systemPromptOverride = speakerPromptToUse,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# jniLibs MANIFEST — état attendu de app/src/main/jniLibs/arm64-v8a/
|
||||
# Régénéré par scripts/jnilibs.sh update-manifest le 2026-06-15.
|
||||
# Régénéré par scripts/jnilibs.sh update-manifest le 2026-06-17.
|
||||
# Provenances : kazeia-engine dist (libllama/libggml*/libkazeia_*),
|
||||
# ExecuTorch build local (libexecutorch*, libqnn_executorch_backend, libfbjni),
|
||||
# QNN SDK 2.42 (libQnn*), Genie (libGenie), TTS pipeline (libtts_pipeline), libomp.
|
||||
|
|
@ -15,15 +15,17 @@ d49b53137f478b7b4ccbd58b9166d83e832c299b681609c70628291c61ae7204 libGenie.so
|
|||
7aeb209ec5c1da8857779b8f3b4c3e7519ed69c6211491948d3c87349dbced57 libggml.so
|
||||
b92ca83a53dab49e212fba411b48fcceca4331b05fe881e2b9220efed45e70b1 libkazeia_cosyvoice.so
|
||||
776f37e43433459a7350dc9d5d15dcb8eb3e17465a28fa78de92559bec3918bd libkazeia_engine.so
|
||||
f91df82c96d1efd3cbe6f7af6b9d261cd5c6f4a929c08bad49a4cfc1deff4477 libkazeia_pte.so
|
||||
fdd730af3daf132d0819b3fc99abd8aae79800aab7a9aad371e117c16bd3a3aa libkazeia_stt.so
|
||||
114795bf0256c380dc87971598707e9e0b745a7fc868f3d1fb460e94cecdd69c libkazeia_tts.so
|
||||
349ba592462113c6e871b9c3ee508727a9e5f154a33905f946bda97535eff586 libllama-common.so
|
||||
2ca77a28421fc0c680e0a5ddf0208a9471268370893f8dd4eba2a10a8aacb4ac libllama.so
|
||||
a207169a2696e8f0f7639090c2882152c734368b5696246a514d63b3ce0769c1 libomp.so
|
||||
d1e8fc79e47fe04ed00aa9bbcc5ccb84d187f0160b1d77ae5bd012aedea9c938 libqnn_executorch_backend.so
|
||||
95d01368b556f0c698cbbb6daa47494c666e0242a063cf1ca830202946435df4 libqnn_executorch_backend.so
|
||||
414794639e2897120143cff80aaf73e830e305357b6c1aa8343de380c89b8c36 libQnnGpuNetRunExtensions.so
|
||||
26f13b297d375e9629eb997b7adb05a4ce9e1c17d306baaefe905563fee43eb8 libQnnGpuProfilingReader.so
|
||||
b5b52b66f3fc5567429024409dcedb006ca6a9115f271689810589b7d79af430 libQnnGpu.so
|
||||
a3bc48674377a04248fcacbdf578a98b47d457821c53e6450ca11b59a3c5ef93 libQnnHtpNetRunExtensions.so
|
||||
2daafb376fe6904aa325e29810480e3d76dc392991d1367ae6ea3508db2b60ac libQnnHtpPrepare.so
|
||||
10eb1923b34ac9a726e4a446b395d5eee2ea5388487917778aaf954e8bfb2d92 libQnnHtp.so
|
||||
7ee72b438c97c13ceb96d18fe255d1f156afbdc4c43669752f349dd5a56e62a3 libQnnHtpV79Skel.so
|
||||
|
|
|
|||
Loading…
Reference in New Issue