From c166391f4990725c3ea9a671f3ee56f10dcbbf40 Mon Sep 17 00:00:00 2001 From: Richard Loyer Date: Wed, 27 May 2026 12:30:43 +0200 Subject: [PATCH] =?UTF-8?q?dist:=20routage=20par=20structure=20(is=5Fhybri?= =?UTF-8?q?d)=20=E2=80=94=203.5->option=20C,=20dense->CPU=20robuste?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Détection au load via llama_model_is_hybrid/is_recurrent (structure, pas string d'archi): - hybride GDN (qwen3.5/qwen3next) -> option C (prefill HTP/decode CPU), préservé+validé. - dense (qwen3, etc.) / pas de HTP -> CPU pur (universel, robuste). Charge l'instance CPU d'abord (détection + universel), n'ajoute l'instance HTP que pour l'hybride. Diagnostic dense-HTP: le prefill dense sur HTP CRASHE (dspqueue_read 0x2e) sur prompt réaliste (reproduit aussi en llama-cli -dev HTP0 -fa 1; le "ça marche" antérieur = prompt minuscule sans fa, cas chanceux). Bug backend HTP attention dense multi-token, non corrigé. Dense a un .pte (451) de toute façon -> dense=CPU robuste. Validé test_jni_native: dense (CPU) + 3.5 (option C) cohérents. Co-Authored-By: Claude Opus 4.7 (1M context) --- dist/HANDOFF.md | 20 ++++--- dist/jni/kazeia_engine_jni.cpp | 106 +++++++++++++++++++-------------- dist/lib/libkazeia_engine.so | Bin 39096 -> 39728 bytes 3 files changed, 74 insertions(+), 52 deletions(-) diff --git a/dist/HANDOFF.md b/dist/HANDOFF.md index d9e45b4..2e37b28 100644 --- a/dist/HANDOFF.md +++ b/dist/HANDOFF.md @@ -3,14 +3,18 @@ Point d'entrée unique. Remplace le LLM ExecuTorch/Genie par llama.cpp (fork ql) + Hexagon. GGUF, pas de `.pte`. STT reste ORT-QAIRT (inchangé). -## Générique : fait tourner N'IMPORTE QUEL LLM (auto-détection d'archi au `load`) -Le JNI lit `general.architecture` et choisit le chemin : -- **qwen35 / qwen3next** (hybride GDN) → **option C** : prefill HTP → transfert KV → decode CPU (le decode GDN - sur HTP est lent, donc on le garde sur CPU). Validé (3.5). -- **tout le reste** (qwen3 dense, etc.) → **CPU pur** : chemin universel, marche pour tout modèle llama.cpp, - jamais de crash. (Le dense tournait déjà sur HTP en contexte unique 98/11.4 — voir note dense ci-dessous — - mais le decode dense HTP ≈ CPU, donc CPU suffit et reste sûr pour les archis non validées.) -Les deux modèles cibles (dense + 3.5) tournent. Validé via `jni/test_jni_native.cpp` sur les deux. +## Générique : fait tourner N'IMPORTE QUEL LLM (détection de STRUCTURE au `load`) +Le JNI détecte la structure du modèle via **`llama_model_is_hybrid` / `llama_model_is_recurrent`** (pas un +match de string d'archi) et choisit le chemin : +- **Hybride GDN (qwen3.5 / qwen3next)** → **option C** : prefill HTP → transfert KV → decode CPU (le decode + GDN sur HTP est lent, on le garde CPU). Validé (3.5 : mono-tour + mémoire conversationnelle). +- **Dense (qwen3, llama, …) ou pas de HTP** → **CPU pur** : universel, robuste, jamais de crash. Validé (dense + Qwen3-4B cohérent). + ⚠ **Pourquoi pas le dense sur HTP** : le **prefill dense sur HTP crashe (`dspqueue_read 0x2e`)** sur prompt + réaliste (reproduit aussi en `llama-cli -dev HTP0 -fa 1` ; un prompt minuscule sans fa passait par chance) — + bug backend HTP de l'attention dense multi-token, non corrigé. Et le dense a un `.pte` rapide (prefill 451) + de toute façon. Donc dense → CPU = le bon choix robuste. +Les deux cibles (dense + 3.5) tournent, validées via `jni/test_jni_native.cpp` (détection + sortie cohérente). ## Décisions figées cette session - **Modèle Speaker = Qwen3.5-4B en variante `q35-lmq4.gguf`** (embeds en Q4 au lieu du Q6_K diff --git a/dist/jni/kazeia_engine_jni.cpp b/dist/jni/kazeia_engine_jni.cpp index f296644..db5aa64 100644 --- a/dist/jni/kazeia_engine_jni.cpp +++ b/dist/jni/kazeia_engine_jni.cpp @@ -1,10 +1,14 @@ // kazeia_engine_jni.cpp — bridge LLM Kazeia-Engine (llama.cpp fork ql + Hexagon). -// GÉNÉRIQUE: fait tourner N'IMPORTE QUEL LLM. Auto-détecte l'archi au load : -// - qwen35 / qwen3next (hybride GDN) -> OPTION C : prefill HTP (ngl99) -> transfert KV -> decode CPU. -// - tout le reste (qwen3 dense, etc.) -> CPU pur (universel, le prefill HTP dense crashe 0x2e). -// Le chemin CPU marche pour tout modèle supporté par llama.cpp ; HTP = accélération conditionnelle validée. -// API: load -> generate(sys,usr) | generateRaw(prompt) -> reset/free. Sans état entre appels. +// GÉNÉRIQUE : fait tourner N'IMPORTE QUEL LLM. Au load, détecte la STRUCTURE du modèle +// (llama_model_is_hybrid/is_recurrent) et route : +// - HYBRIDE GDN (qwen3.5 / qwen3next) -> OPTION C : prefill HTP -> transfert KV -> decode CPU +// (le decode GDN sur HTP est lent ; le split le garde sur CPU). +// - DENSE (qwen3, llama, ...) -> HTP contexte-unique : prefill + decode sur HTP +// (decode dense HTP ~= CPU, pas de pénalité GDN ; prefill ~98 vs 14 CPU ; pas de dual-load => pas de crash 0x2e). +// - pas de device HTP -> CPU pur (repli universel). +// API : load -> generate(sys,usr) | generateRaw(prompt) -> reset/free. Sans état entre appels. #include +#include #include #include #include @@ -13,38 +17,47 @@ #include "ggml-backend.h" struct KEngine { - llama_model* m_h; llama_context* c_h; // prefill HTP (nullptr si CPU-only) - llama_model* m_c; llama_context* c_c; // decode/prefill CPU (toujours présent) + llama_model* m_h; llama_context* c_h; // prefill HTP (nullptr si pas de HTP) + llama_model* m_c; llama_context* c_c; // decode CPU (nullptr si HTP contexte-unique) const llama_vocab* v; llama_sampler* s; }; -static llama_context* make_ctx(llama_model* m, int nctx, int nthreads) { +static llama_context* make_ctx(llama_model* m, int nctx, int nthreads, enum llama_flash_attn_type fa) { auto cp = llama_context_default_params(); - cp.n_ctx = nctx; cp.n_batch = nctx; cp.n_threads = nthreads; - cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; // t4+fa optimum decode + cp.n_ctx = nctx; cp.n_batch = 2048; cp.n_threads = nthreads; + cp.flash_attn_type = fa; // ENABLED (decode CPU) / DISABLED (prefill HTP dense) cp.type_k = GGML_TYPE_F16; cp.type_v = GGML_TYPE_F16; // KV f16 (q8_0 = -40% decode, mesuré) return llama_init_from_model(m, cp); } -// Cœur : prefill (HTP si dispo, sinon CPU) -> [transfert KV si HTP] -> decode CPU. Texte généré. +static ggml_backend_dev_t find_htp() { + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto d = ggml_backend_dev_get(i); + if (!strcmp(ggml_backend_dev_name(d), "HTP0")) return d; + } + return nullptr; +} + +// Cœur : prefill (HTP si dispo) -> [transfert KV si dual-ctx] -> decode (CPU en option C, sinon HTP). static std::string kengine_run(KEngine* k, const std::string& p, int maxTok) { int n = -llama_tokenize(k->v, p.c_str(), p.size(), nullptr, 0, true, true); std::vector t(n); llama_tokenize(k->v, p.c_str(), p.size(), t.data(), n, true, true); - llama_context* pf = k->c_h ? k->c_h : k->c_c; // contexte de prefill + llama_context* pf = k->c_h ? k->c_h : k->c_c; // contexte de prefill + llama_context* dec = k->c_c ? k->c_c : k->c_h; // contexte de decode + llama_memory_clear(llama_get_memory(pf), true); llama_batch b = llama_batch_get_one(t.data(), n); if (llama_decode(pf, b) != 0) return std::string(); - if (k->c_h) { // mode HTP : transfert KV vers le contexte CPU + if (k->c_h && k->c_c) { // option C : transfert KV HTP -> CPU size_t sz = llama_state_seq_get_size(k->c_h, 0); std::vector buf(sz); llama_state_seq_get_data(k->c_h, buf.data(), sz, 0); llama_memory_clear(llama_get_memory(k->c_c), true); llama_state_seq_set_data(k->c_c, buf.data(), sz, 0); } - // (mode CPU : pf == c_c, le KV de prefill est déjà dans c_c) llama_token id = llama_sampler_sample(k->s, pf, -1); // 1er token depuis les logits de prefill std::string out; char zbuf[256]; int pos = n; @@ -55,44 +68,49 @@ static std::string kengine_run(KEngine* k, const std::string& p, int maxTok) { llama_token tok = id; llama_pos pp = pos; int32_t ns = 1; llama_seq_id sd = 0, *spd = &sd; int8_t lg = 1; llama_batch sb; memset(&sb, 0, sizeof sb); sb.n_tokens = 1; sb.token = &tok; sb.pos = &pp; sb.n_seq_id = &ns; sb.seq_id = &spd; sb.logits = ≶ - if (llama_decode(k->c_c, sb) != 0) break; // decode toujours sur CPU - pos++; id = llama_sampler_sample(k->s, k->c_c, -1); + if (llama_decode(dec, sb) != 0) break; + pos++; id = llama_sampler_sample(k->s, dec, -1); } return out; } extern "C" JNIEXPORT jlong JNICALL Java_com_kazeia_llm_EngineJni_load(JNIEnv* e, jobject, jstring path, jint nctx) { - const char* p = e->GetStringUTFChars(path, 0); + const char* path_c = e->GetStringUTFChars(path, 0); + std::string p(path_c); + e->ReleaseStringUTFChars(path, path_c); + setenv("GGML_HEXAGON_GDN_PREFILL", "1", 1); // sans effet sur les modèles sans GDN llama_backend_init(); - // CPU model — toujours chargé (chemin universel) - auto mp_c = llama_model_default_params(); mp_c.n_gpu_layers = 0; - auto m_c = llama_model_load_from_file(p, mp_c); - if (!m_c) { e->ReleaseStringUTFChars(path, p); return 0; } - - // Auto-détection archi : HTP-prefill seulement pour les hybrides GDN validés (qwen35/qwen3next). - char arch[64] = {0}; - llama_model_meta_val_str(m_c, "general.architecture", arch, sizeof arch); - bool htp = (strstr(arch, "qwen35") != nullptr) || (strstr(arch, "qwen3next") != nullptr); - llama_model* m_h = nullptr; llama_context* c_h = nullptr; - if (htp) { - static ggml_backend_dev_t devs[2] = { nullptr, nullptr }; - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - auto d = ggml_backend_dev_get(i); - if (!strcmp(ggml_backend_dev_name(d), "HTP0")) devs[0] = d; - } - auto mp_h = llama_model_default_params(); mp_h.n_gpu_layers = 99; if (devs[0]) mp_h.devices = devs; - m_h = llama_model_load_from_file(p, mp_h); - if (m_h) c_h = make_ctx(m_h, nctx, 8); // prefill t8 - // si le chargement HTP échoue, on retombe proprement sur CPU-only (c_h = nullptr) - } - e->ReleaseStringUTFChars(path, p); + llama_model* m_c = nullptr; llama_context* c_c = nullptr; - auto* k = new KEngine{ m_h, c_h, m_c, make_ctx(m_c, nctx, 4), // decode/prefill CPU t4 - llama_model_get_vocab(m_c), llama_sampler_init_greedy() }; + // Instance CPU — toujours chargée (universelle) et sert à détecter la structure. + auto mp_c = llama_model_default_params(); mp_c.n_gpu_layers = 0; + m_c = llama_model_load_from_file(p.c_str(), mp_c); + if (!m_c) return 0; + c_c = make_ctx(m_c, nctx, 4, LLAMA_FLASH_ATTN_TYPE_ENABLED); // decode/prefill CPU + + const bool hybrid = llama_model_is_hybrid(m_c) || llama_model_is_recurrent(m_c); + ggml_backend_dev_t htp = find_htp(); + + if (hybrid && htp) { + // qwen3.5-like (GDN) -> OPTION C : ajoute une instance HTP pour le prefill, decode reste sur c_c (CPU). + // (decode GDN sur HTP = lent, donc on le garde CPU.) + ggml_backend_dev_t devs[2] = { htp, nullptr }; + auto mp_h = llama_model_default_params(); mp_h.n_gpu_layers = 99; mp_h.devices = devs; + m_h = llama_model_load_from_file(p.c_str(), mp_h); + if (m_h) c_h = make_ctx(m_h, nctx, 8, LLAMA_FLASH_ATTN_TYPE_ENABLED); // prefill HTP + fprintf(stderr, "kazeia-engine: modèle HYBRIDE (GDN) -> option C (prefill HTP / decode CPU)\n"); + } else { + // dense (qwen3, llama, ...) ou pas de HTP -> CPU pur. Le prefill dense sur HTP crashe (0x2e, + // bug backend), et le dense a un .pte rapide ; le CPU est le chemin robuste et universel. + fprintf(stderr, "kazeia-engine: modèle DENSE / autre -> CPU pur (prefill+decode CPU)\n"); + } + + auto* k = new KEngine{ m_h, c_h, m_c, c_c, + llama_model_get_vocab(m_h ? m_h : m_c), llama_sampler_init_greedy() }; return (jlong) k; } @@ -122,15 +140,15 @@ extern "C" JNIEXPORT void JNICALL Java_com_kazeia_llm_EngineJni_reset(JNIEnv*, jobject, jlong h){ auto* k = (KEngine*) h; if (k->c_h) llama_memory_clear(llama_get_memory(k->c_h), true); - llama_memory_clear(llama_get_memory(k->c_c), true); + if (k->c_c) llama_memory_clear(llama_get_memory(k->c_c), true); } extern "C" JNIEXPORT void JNICALL Java_com_kazeia_llm_EngineJni_free(JNIEnv*, jobject, jlong h){ auto* k = (KEngine*) h; llama_sampler_free(k->s); if (k->c_h) llama_free(k->c_h); - llama_free(k->c_c); + if (k->c_c) llama_free(k->c_c); if (k->m_h) llama_model_free(k->m_h); - llama_model_free(k->m_c); + if (k->m_c) llama_model_free(k->m_c); delete k; } diff --git a/dist/lib/libkazeia_engine.so b/dist/lib/libkazeia_engine.so index 2de8651b25d634835e3bd6e67e3914eb231ee36c..2869b49673ffaff9f1219a077d839ca83e49014a 100755 GIT binary patch delta 13511 zcmc&*eOOf0x?g*57>5CA_>e(C8StawLp~G%KhR-Br7|O=g0e&q#X>{W^rkhS9@8|N zX;E@~Ov&m5+~|;F9XqF6_;I>)kLR4RtYaOkf$Ti=aSKnBjG}XYYwbO-qp_anJW zl?PF<(=odzwf4Wg9;*UWxlnC?4-p4shS3dbgXmM&qeT@-m0qi81IiGYU7J^{VdLT2v?&Am%roX zTfF=`FE8?P1kDRDNoHPV^Kv0C%UKz?3ciFn>m#HZPHyDo6TJKjFJoy&ph-&Rj5cyq}jFdFkL~3olRb@^8HSikCiKzD;k}nWXo4`3Wz-$tp7O%gm|S zxEYSYZ6*z*PBxoo)6s_eBi7eeSFRyT?_RTUHCbJ`di8@3RnmIH?YgCRFRw0qh|UPk zOs%e7wtCsp)ekPOtX{fm-O{RmE3aL(oD5@YD=RkC)>f`rPcIs79=UG)^2*v;vSMTH zs`Zt0U2q018aau|BNw^iLhh9yjB~Y&bV$^CLzx}hRNJ14wk`c~=P>^C3b>EUJ&ZrA z@#lu|=QZ9lj5n`T`{`jk#wY%P^;N2(cbGx1#`h2716HZ+iMHMS{X}ZKei%PN}vk}gjk0uItx^X3Wo9i%xl&X zqyLGmnrs=y*J*P4FYr01`121fLEUL}VDk|br2x#1TIHoh;FAPCUEq@iK1blw1m2(7 zB|G*-x}dNM1{nfhrtvZyVq3|7RSCS7mbt1%;I%x$`Aq_kuC)0_rA|<2TbUc|6nGtH z)WRX~BLu!t;Qg;Drfw4WprQ75(kv(pLkiL^@WBFqOyENVzC++g3Vf%)hYGx#^H_g2 zr8fwQbAo|U;5`B#Ch$E1A1?5{0v{pp{Q^Hqi$7KEQh_-XB^c-h-X!owfsYpWXn`Ls z@MeK$shC}Se2bunQF&gb3w*4==Lq~*fwv0$IDsz^_&9+tbZPO&p4coXN(6&=fiDyI z1c9#-`0)Z?Bk&Uhe$yb2=btDj>IM~rOcMB=0)L~xI|SY$@QnhWDDX`JpTcWbA^2p1`l`l&vZXG!%d8D&Ca!fIw){*l{ zK&OuMD&})$9XT3Ma0+FhzFkMo28JAkBG3q?PPZQ2(Q?z2yC6BFLP9)JWQUIIk&PZ5 zIUwhp*O4Yg-=HIJD*8qpIi+Bec$7@^^Flz#%NQm=-=ibn1f+kXBkn+Rua0cj>Cx97 z9c$j8qhFgcT;^!Ivhaar4_B^QHo0=m%2jJBXP9x?IoVuYX)Y+W72T9S&um$czbMf> zdA9k%n)Rz5Tw|VVw$#*Cu2@xFZ7wJ-G^d%DS61MFWS(1iN1}lqjCq278XZeNG>>sT z5c~IldB4zm{=7wt=fQZ{hV`|Ttaq53Yc|vlcAk7)w|B>{q@D3!x{f65knHz)eZHN2 zKHt(SKHqNO9N=$%@cCM=`h0D;@44jjeS6vGYe)U!k3OFb`rDu{2gX8P(eLxkNBtt& zOt|LrefPc3X9oWRD_oD3!Zcy-a4x6@9>7b^gCrQsWF)+d7~Lv20Rbk12j%0 z<5+sLv%=+4pee*AZY>?0t+&6h5?=eVvCV>B(o44Oe{!Vg}^3YBk(Bj zWnd?;1E@z5`v#Z}jK+bb444nx39JD&0=EM@fbGC^B)1ECr0sNMvvizfI*`0J0Sl3= znt^U4r#@ig96~a2ps`p9Sq@AGZUW{2_W-v5n}Eyn;4pBZjgTH-8E^nt12p2`QwN*? zteK1D0Uia`*-`00r2%*v*bM9ko&|22k0-#%Xgjb3_`(7h0#g=ZN`dQ3u=XQiQwryR z$M6Bd1I%9vITW+DjF2i|=lvKAcwjZ*2PF3zw825C_d)z5=RoDyTC5k)ybgv4npot^ zDkGh58JD{}PW|~1mi*_$6kr)ROd!9|npklc595l#70~A_5%vOT{AZ3xuv>5!3cW() z2j>Rhn+cdwM&yUO<-(A$=~QkAcv^Qrf6(x3h`c-`4s0lS|Lh0nKm7#pM>=u$Bx1@{ z8*@R(SThYcsC!m#s0xWQ7ohFZPd~{I*6N>rVnM?J);I}M`V%ABI@l1pXxtXOc2dxm zIS!|c`5|MC2jyo2CNL4Y^+|M5V#4fdRJZ(0GuFjU-t4DcFrQz5t~!anl6bSO57o$I zdM$B&kX440$yj&#YT~F61R7c{2e%}Ju1U($HK7_gnYJcP(VaupI+=zhCm7h1u|!b7 z#7Se)Q7xQ2H1kZI53Xb~T?BOvuMRCEQ?qoia5CMSJZT(D2+W$TuK!9wr4u{`crQ6A zs2|l1G}%I(DbaLn%J`sJ$O0^jd_&hJMbITFDZ1@o4y4k(DYJDfRk%~>dnvY5BX&PY z(>oTZ~EDUajcIXt{HNg^)Uc0gc&A` zWbtA_KG9?78aQpD3sWP5mVn6zL+IqRN%YCov4&I+?zb6isV07Nx!kDh26Q?8VJ`0cXZWU{?X|3*gdc(r+{7>(UL_Ds$+JDf4w( zP_4R!?wB%8$MWu`g|v0b5nTx~ru$Y}KGmjcLN)Ss+B$WH&V%ZKBKpM4Ya39h;8#5nsT|?`$O!URf7~PvtZC^^?$($ddABkxyL-Y@}T0%3j z#?o2SVs*vPon1!nn>Jt9glca&ZJmZuQGL0Bel>0Eq%ZcMYh1B0A#prUO@Ppe7cKiL zs9)gm@r%yMG)*}Wc`)kPXoE6A3W+N#;d$=Ezn%N&Vw0Vg%=ffXS|IbL$5N9Q?c1MxoOOWH1kc-`jy`G2T7JvD? z;>?k(M|z%Dre{m!HLOLXH`ICryU+O_u(JdcClXNWCFO>eo{hxoQKDPBl&IEj#nk!+ z>-T}|oSm}Ot8cME7YW@cJCsqaY;fj(-HX@KyKtzDcVs&gNL^7oj9j>Tw#m-dfSwWa z+j`bny*?Ltd!EH^-*n#J^J<`^Ea`ds5xzWeE z6a%^?NE3aKk%UN$Sy$C{OU`oDySEsClCOE%5p*)-3c z?inHRCy2u|Ta{h=v6Jy6#90J;7C&_gd5t-UqeLXaxEDe1CINb;I6HzRu9fKOic&#*CX5oirdufKwHba49^^QSXwmfgjq`@~g@XXcC;VlDqMYQzY7TI!jTY$4? zuI&6?k(2rqDTys_INGz==^jzQW9R#q$QxaYF?D_8!+8LClu@nSh+@xXaVN zP~tuAROzOf5Veqz&to6H{?DZA*_nJvU<4tdpEQcB!SO0vG{#vE=kUU`V_iAx-ZYL9X_E zC$|j55pUOc0{iIJQLs1n`KGB45X~RJ430Vb8Xf|EnuIMwQ9kv~HrM=xg+FWXUg1xGS&zsweF4E^1ld83}kEcq9(33av+Y&6We0Kc9r zwPs5zsM8uJmD5+Prm+9M=G)h!XyJ)~tfn7X?~?Fq$-En-GFqBY9p`=ONYdGv_|8;hyB&fCz>Hg`s-{1UETav%lE;=dlj408fo=n7q>pi>=6!D z*gLeoSp5lB$2*PH_p-V{t?x!10lA{p-?0gUT;2S33+og~v7NXjC(=p-8?5g(iSsPQ z5V@|X4#q4Gc3~CSStuT7s07?B!s8?Jyte2(?`E{|r>!k?tt}?_-@qXAx%Ff7=suen zziq-0=Y76?*9}`3;QFiWn3U>fFWkJ5*Np2D^1R)ML0irg=jXtRsu|9*69LX2!|*h~ ze&7}QTK+-Rq)Rtw(r$|lQcX&3o>rBaK&o;WQrviA#`d+XyC2RUkR+7S0yR%?HDI&tDt z)N36+UoXi?cFqFN>y|9ncejp}Ox7sHSdBpvv65^x6EQi#qiyuc;+sZUBNU@ywCwfA z6npq$T7O%%{j55gG~C}Y=r1h zvRAK|c3(o4>qoZZ_Mxi%O=D%R=?>C@t=bxUtNLVBc(NpT*n=FQ$E!R_ilW(t^X?7P zyspA@CyrI8d;EZ}n2phg1b78!mC<;b%dpFqM*qo!sSkk-OOEV_$c~N!q3X037!>0i zMgO(6--l@*-KwJ>6*}Z91AXB3kLBJ;t_4NKQffQ%^tpzsJcLhRLT%RR&Xd53!ZaR_ z8R(6L@Kj(HeRpw;sqQ=3m<8$QX1DGe`6shG&Q)3*DrNAI@^%O#otvhPWS-1NdTXRQ zQU~xPZrWa)PtV`6PW4f;3ziMJ*I91YEO!8R(MRswEJeAzcRnfMFBW!{B%8>)vM~xO z9txO9Jti;_;qa%DjKGd}WMdzdO4FpBG^2ELp!K+H?4+-jCQE1NC#B=1(=O@mY*~7T zPG7Q7CLyjPOSa1RbCPLGH%nIf>e3i{-qb_j#GvMGm<*d$$pQ+gCQk9E|Qy;IME|uc}&c&i`VKCR#P=-7GvqY;QdyoVfYnwe+QY1;$IcBF1K zBHkM?HsB?K9q86}pquT$I_$i7JOjh=z)86MHqx12w!z7t+e86J3pUICdY#Q8HC(lU zMrcMsz!1Uc_@A#cy3Xb@o{Kp*0Q9+DtPYkE<4+Cvvq`&6jqY}Tbhn`y%O<}C9&vrN z+9lna_nOdrJ5LGi6?@c_kfk-x0nWr7fwvCdKrX>%*bs@DO)@am_2q-1vYp!;e@?Jb zlU%z$$?eClx|mxBfQN9$g^{%$1)qLLk+oeqQam{Bp}{`fwz3d56YUkJGWj0drS-8J zxR>=2MAk}y%+{_tef-&RGF53{ke zlyq9=MrE2&Ij~YN>S`Qj#GdSY4!2pX+1vpv5p44I{j$w`&87e--QwD?ZiQ5?F5TmU zM(wuKiRu*YOy&c11**&NIq)C2{j(tL`Wc%VHOHRSY(4~jL~}Q6l=pw?dVRyR5$dYP z-z=C8TlG&Ab-2%fIQ0(>S6}eUHg3)4J>Um|P1pS4Hg-&)_6|L%8T}E656y!E<;@#3 za%eln;om8*Yc|J#f53eR-={DiZWjLaa|Js`2Ae}i#nCx@4x2T*Ry!(f!wiq-{fyx$ z^+ms5GsNHEH5?jdCs}`D*IwZW;REi`?DhhkxcyGrm(Bi(k)xt|IyYj@u?<%b#Mk*j zqvNv#8+Bt3&Pc6hvmUrnu!*1fvo;ZYpi0eXCGdWQ-d8rBjpv3gvu_Br?v;$`G^}oFVBH?cm_To;%a)GP`nt#LWjiFJN2$yG20zK;#nFo|jz2jV z$ABAtFz!Kn@nDQ!Zt#O~=dYye4#wT!haZeRN=I&VDL))XeNeVlGMYvbF9{-7{RiYf zyPmB-C`GyP`g6qh&2+NY2s?qZth|ukzU}@5RW*>Tsq{&oYwBc-$C^!ceX>msu&c54 z&ny1y?XYF0s{8YR!Y30y^Yc|p08Yg1!O1jkB3T=cFJt~?ukZbdy`$osh1(l7`*7eW z-0ah;UQfc}ekHB#B=o+WI@3)>14c_}^D(oe-_3rG#A&4+-;Z3Icg~XR?y%H6udzJH zX_n*o$4<5x*o|-X?yxD)ZB=z#&VqbckY_+XA;_~KcMGx&vTWdebNl&_69oBYI|!RV zEP`Av$VHI13i6$h4-4|$kWUEmeUQ5axg4?_JmkO%$O(dMzaNB6Al5)G7vx&VTLt+c z$cF{_VaO*0`BBK-g1i;795Up<4#){ZvOVl^5Vj#AY%k<;L4FGIRzZFS@?k;V5BY>3 zAB5a3$cG`zBZnOL4djGD8SCE)!Zt{_!(M@0F33k9Zx!U%As-gx-$Onj$ZtXJ7UUC< z<OFxm%EB8yqkWIiQ1_pvin&7(m!m zg62JOh<^4&P|+-W>}3}|#_+3Pn7nwHTro^uJxqQykbeBcIQ%_(>E5(3_YG>N$kqI6 zqQBmohR<5>?v0ZRj?l06&Yn_$576w=fb7uv`{A1JLB>aG3M6cPS=;H|PmT*KMCxZ3 z{;Y*JJ4bgtxzy><+sy}5#lHyc*-rdWL2P`GX^xhBah|`9sN%J7{UcPz?HG5i!`17KD$U{Hmd6 z8nO+ICRJ)Q(Pa%$)bg}BypjEx1TH^@gwng8P7NFWzgG6pUq77|mW~f9?DAtB{pjhm zaR1+v`~Qbe^ImkiRJ732eQAaQjV*YJKD93^Jo>1r^#2c^o?!a%zEp$%&%vV`Xn12*nE%h&tqp2x V{EDJ6JIF76P0u`GqBRZr{{gOQ&kz6r delta 12787 zcmc&*dwf*I*`9Mwve{gS8xoRm%Z*#^2@sN?0hT3lD@wo=xoHwYAZkEvLMrHnfP#RU zbs|Qj8dI!EBz1AIO)ZrKX&Wu|w^X255jT;xTcvHRL9!UwJ~QW>WLWg~`@ZjwuLsXO z^E~goGw+!*GiT1(X#cs~dPJ^Clv|&(C9kzs)7@~mFp<_ z8DKI$t=i2kH)FA%<~Nw*GUDr4Z&M4@x8ID3DkYUhEu#f0gR%x)T7xdFL01diAdOw} zKa7L0>t#p5UpB2|A-0X-^cLL7+QYHa@s;$tBvsIOB}&@G(*~aYil--d`Y})c$x}bN z-Y-fD=4m2Nr}8wPr%QRdiKh)b?dGYMrv_T$A09)-W~u)d3L#;(?ctJ_i%~QzAW9m=(_Ee|pzEQm;`yyS zU&ZsgdHO6*pXcd8mOf{UmO5C5W+Hcn8!qwm8dqze2GVw8tktCru5PV!x8iOLCWDA` zuG-lrwY!sm2$>#Hpz4*HzD(2iEl~A56^K>SJ2YK$gH_i4Nx1Qn9}<~97C4md42)J_ zqPD;&`c~ku2V`az{+Tc(;^5B49@Aqq4oXS~vH7Bqp$u@ST<+&gWARSWa2PO-)RHPW zE^Z1PAM_x#2hESIs3=)eQncpbm1Px0Ysxm26m2f4DB8Gb1Hp;^=<49P)DS$;zi8>& zEo}$MHdLN`_TiHaxJYjIOz@ z#Hvq7MVp8^U#!g=U0=Qu@!=@;(N}7ExsP6<>Fa!SyQWvFx|Qe+)Rn0XRQWhmty1;f zK6R^9UMa%@E zud_^P74#&v2v6GtJx$Q{HCvJhNf-2vf$~<8AsF16K}N&y=|8#!U0V~ltyj=7w+9Zp znq~Fizy8B0=<2lO*N+J-7!3Wz35gc;06~uvbo~Lu%!z^?D3s3>^dLdcwF-t{!C)5j z5J8_U=z|2kK+r=4eX*e5Cg{bQZe?9$5)9>nLztki6Lj@itr{x@JzOYXCFl`?zFW|T zXycEYYXn22;7}*%QG#AC=+T1SDCk24-7e^CtyK@)|5m{;T-SkZf<8jfI|O~CpgRRU zM$pd-daR(kn9jx@y9IHA!7Vt%3wpPpCkT44ppO!CazrJp{?USN6m;tt!C(>$V+B20 z(8mdSoS-KQdZM7G2zsWVr>eR-|8fOGrr=-}^zni|ThJ#6dV!!%6!gV{o+aqTN7VND zEtn)2$_0mPL0>26+D@5UD+N7QC|@P$lLdXZt}np7xm_^S=mzL_2zs5M>)RP-_If1~ z=|JHQzlsa~J=roWEewk+sw{^;!rT{&W3 z;QB3PYbUNPtK3B6;zv3{MqcrYSY5WZY(q&!ddY^;a%?h6H$AeUjLuA0ZQx}m(ZdN} zIG!K1OS10rc)dM6UhnR2z22+9a^Uc9yxykkUhiR?k6!h9qoBQs{C&tjg#53rc)fdp zyHW1hUa$9IvUf6?+3qpA-xF38PP`3MPueTKSnqjN8qR?>|j5wv?${p-YWF2 z$iz0`B(NBmj~Ra$7%~;T0XzsCf$hQ}U?%V|Fdx_rTns!v4M70g%(z9{EDE)p%cNMIIlo)sB0GR43O;6C8xd_)iI1)4(8tux^Wte=Z61-ciZw}2xS zqw9gGO9`pNHl+1Fa9~vt#sjt=c@LmYps4~jVE!7^3tYJtV*=Zd)OCbJ)*@589*)48 z4RFMuA%vzU#S9q{3tz@?Figc6O-qu(ts$_W3;Dq;g)9dxf-7q2a!u18+RU~Mr%|Yt z6&a1Q7U$taM9hTz(74$lBjc#w%Z48tSBAvK%?>3WUHp`lJBG6@k&Gh+w-l`GFUYnb zmvFih>HW!(1|s3MCDBKd6A}`U%}vt6%nvo?hm176EdSV#dE~3b-i1dovWZFb&&hKQ zc4VDNG&yCap%2;CWV$&e#t6rCv8M3eKd9Kuv9k9VAgw-nb{mQrqXv($Hr8!*#|SLNk7=FV6zpC z+(=C}97T3pDoszDYUoAwXe!;1W(hR=VSr-#R?_il(e!NED8o)D`_kx*G)wv^WRJlp z-ZD0iMU4g0!!^1#eXJo4)7F_r4oXf;{% z?(s4Hy~q~QyT%Xam5xAd(O5ttSV?kxhv=HRk!L$x&E|2=7@;UuyVMRaQROoIt4SxqtBlRYzVIkJaKFc|nixSTzD z&Os>UE711{h#mUOk{&IXw~$ENVuBC8xK}d`$7(eX`O!E}hGE^tY&52Hdig;go*18a zNDhfDUW`@d%-M5)I{)#XKe=%6)BIr8U7K*O#LZ^kx{Uv#4LO#e3q!ENVv-quNB^0V zBIVOzxd}mjc*kMiy^MdNGjngB%$iyW>clHf0rJMzU21cBhsw2lb%Ti%jaJUa#wlmp z(iD@8{yH}=Dn&6p5rJ;d3^O!N*KHpuD?v14@(}Ac3TZaJX7Tit`q_vy?c7yxUdiDb z1-LSM{Glvc0;yWKK7!P4!s$l2%|LI8^h@`fW>4n~#a3mQ-r`cCn%#Se$F(bK|8J;e%2P(4*!-1QIHq#3$VFRfblj$ z-sS-u&g5DdG>o~UtAY5eTw|8W(OxCC8G9f5=6SPCMd1Bk5%_E?2{X^}{wONnk~ z{2Cf70d^`wn@fRhH@uS`{TuQ0c&*+j_bAC`fmr@Gv*&@3s)c`FMt->)Bb4<*$4SNH z9*3*i@!P!49ly)#REW(z%j{uS>k5z%{BNM2%yW+-9@tv4P!HD4<~Tocv~h-fdLB6P zw>1PW;@+8fQeeb}ECwKh1M( zteR54Yg>yeAf%;!@75MqaBxc=D)gWMjM|6}nFSKr+^Gy{?m|GuCX0vN!Yk;NRMX)PKW-KG?s`x?P8Ef{u}65jl@OlqlVN|}5%oo<jYZOuZv83H7>y z&7Pmo*QSmNJiux}Ya{6CsjHKo1jB^*!EtY1)A1c6d*fu173)XV?+GV`*B&?1C08H+ z$b~UgPD`c@mwrSan>H)06h7w-X3t`b-}9mD4n;Swf) z{e?>z)*J^LE7OHM8y{KVSe^L2Z$mU}L1KW-rF^>82ZcnjT_h8X$Dli4T_%fEi@9?AACW>#OgWNRDDeUm4` zkMsuQFy6I8Vz(+;iK|+82|6LlyNE6h~_jH+%r%SqP|)hoW!Q2 zd!uA?Tcsvk@3VpHH()#IJSf^UZ}3qBMY3-oPV zz*PeYi~rVJ_!8bs>pv%_U&98{g+@Do`1lp!nP~Q0i9vTxR>)CL4)ORwWK)mLlozo8 z;pWrZEO!q1^k5Lz z`zN_H)rN;q?e(qiHucPxxXu=EB3CWk246N;J29#8%tc_to`4heMK;HupXIS!nC01m zLVU$XBbBr>e|YdCP!dth*g1;^+!+_m+qmHMz96(~$D_|N?ao#y-5!g@SQ>f5wDGUA zJY9&$GCITdIk0qFj;;8FpY7jaxGG=|@G2cM>t)sD)yRGqyQRWkb*UV~U5;xm?ZA_c zuV$6Y#*#@k*3cYUGB;cr=Xhjpu-}5>bj38z#Dne5!ZUx!$7av&i)PPp+4yA_2I{Lp z)0=)GpJ*(Wr=8RFQm?m=RaBL#n0n@0ny&C0^IqfeQOen9diw4R$xNmBvDV@w#q@LK zzTO+Rky^YncH_%Ycj9Me&z#TAo>(dH%b&=k5bu!Pk9=bGOq7QFzWAHRKiYO<2EB&z z>Z%{!%)0dpaMoT&qGDok6(cS-1bd)A6**$>ZaIAKrcVg!C7#IpNE453j1p>!ULrLa zw-a09XLx?mmic=U?IbS1X2B58Hk*5MGKp>T4e#503@lEv;jJoy=kq2yiV8+bQB{$O zsT_?YvA}UAp>u*iHPeR{&5a0GOhKO#k3Oc@wHMH@7UWnX!W7fccw)=N$s&2jq?p)M zvaJYqKM`(RVKTS}BZ}E;Z&OTxQO3Qjf#{Ek$EZZ@y{Zgv?!|h>{X@CGb51AmME#I7 zvGs7oUFwCEGev7F& z6c=+1emT<6-@DNBM);^}WE9#a&Ap*&ukQ|0Ow$zOm*!qCdVOfKOy}HFBexppC-?kW zHjQ!oqOed(uVImv9Jtt~abncuXZI|U+dUu%P{Y9bDc;1qN;f`hF z{mJiS6MpkvmLYAUuP)o_U)3&~PSW`0DbgS4UCT#F9gg+Ob7biq+I0UGxi8p}P*g2T z%jjz*Tctc2zhbymnjys16R*bQ%-~m)5UgGiK8nMOamI$8J(5aa&1{8UT7qw>io31l2D9|AjRA z+s0_Too(eSNAa?3PeBmGy+%@tRbMM_^*YJnEM6c!P_2wj?EcE!GPiYV4GkW!Imy0> zHujY5@3HPNyezxYcw7)(q6cheGW%^3Z0tRy41pN&9O`D>%p#jaN3Kn=;?0ZS-IA_8 zQ!Sn$n1N1h20GactisG230H=(>{@W@KJBjW_c3WcgMmW?pZL#i^0}K0*ZzhS&B-5V z6r8qydXv*lKHmhW!Qm@H?YE904+l$0ogek-tH~{+y9veEYVro~XO21R9MXN#Z-nA? zd`WPY9#)rx30mUx(9j%E)@cUV45!pMy_mb?gG}VRZzO z^^!mHwW`kBTfW2Dn8%$rYtEIxtvK}tJKy*Y9}>@f9?*Oq0POR@CM`0bazZH#dFp_xqgEd_Dx87JOb^wGxD3ZTl3okd>W@7q&5ETTb%5rRoUEWtLF3=5Z{{mowm;te0+Ljt>&{1 zxKZ%wocoRs0QCnb|hshZ-(yEz^l-T3%*%f5IxG~a#kC6pKU#b?0T zl4L}E;=cGIbl-ikOKF=Dy@cqDJ z80o_M40~kErtMEWn4p?QBx@#P^5>d)9NJ^Wk{owEDf?ONET-vle{a@U3RBgwWhcBd zM@Z^~i2=srbYW*2JBF-}zu_&!ZCCxtE&es7y<@q5h~^&(48zGj0OCOt7W;&>zJt&Y z_ZXs@?(Z`tE@^CylZ<=W=N9Z%YDnTV$CrEZB&$8_-?KcX>S?w(Irbb2RzLA6*&g-{ zYz?Z7^K0NI1bzd&tDif=@Um%=39k#~ond$(#gB~x950|Ii@-yz5LO5RJ{Op(1&+6Q zQ-i>Vfu9g~40xBo6ToF@puw@=2?9?Bw+P&t4q=5LOaQMIcs6*0z;6dXA@Hf-T>`g& z%kn?~Gr20)G_zguovM?-F=5xa>C&z?0w!0^b8}(K(*uVb4HV zp$p*8fmaKBKX`+{4}hN#_{-p30&f79{Raa0DR_dwo53v_x3W?CGYBh$0&js=3;gHc z4FZ1${Di>Y1@99032@mk5WsK269oPNxJBc9ntTXhg(}b^&%Q$E*96km8dJ#xeCcKf zKFIK6iVwfbhu`bN%YC@XhsV(OYcj&>@hdJnun*_QZW?IIkP_)+TdeGSlP1nQ%L(5Lq2 zgy~~Q$944d{(-@#4?_F}`U@F>iFi+BMj%+4zK|Z~`zQYht}Y{NHg+cL&@q=TsLL3v|7ouNzel7lq|eo*2ld~gr|1WD*+J$O z)z|zq4SF#