From 67de8d47671bc96717e1f1bb9433e5d8212a89c1 Mon Sep 17 00:00:00 2001 From: Kazeia Team Date: Tue, 14 Apr 2026 15:08:31 +0200 Subject: [PATCH] =?UTF-8?q?LLM:=20enable=20hybrid-mode=20export=20via=20nu?= =?UTF-8?q?m=5Fsharding=3D1=20=E2=80=94=20TTFT=202.9s=20=E2=86=92=20113ms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-exported Qwen3-4B in hybrid mode (prefill_forward + kv_forward) with num_sharding=1 after discovering that sharding=2 produces a multi-context .pte that the LlmModule loader cannot restore (error 5010 "Context group 1 does not exist"). Single-context hybrid .pte loads cleanly through the JNI runner and the auto-detected eval_mode=1 path. The peak RAM during export hit 49 GB, which is why sharding=2 was used originally — the /swapfile (192 GB) now absorbs it. Compile wall time with sharding=1 + hybrid is ~73 min (two graphs) vs ~30 min for sharding=2 + kv-only (one graph). End-to-end on tablet, same 'Bonjour, comment vas-tu ?' prompt: Before (kv-only, short prompt): TTFT 2865 ms, total 4034 ms After (hybrid, short prompt): TTFT 113 ms, total 1471 ms Gain: -2752 ms TTFT (96% reduction, 25× faster) Response: "Bonjour ! Je vais bien, merci de me demander. Comment vas-tu ?" Co-Authored-By: Claude Opus 4.6 (1M context) --- executorch-patches/qwen3_4b_decoder.patch | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/executorch-patches/qwen3_4b_decoder.patch b/executorch-patches/qwen3_4b_decoder.patch index 91b6c34..e7049ef 100644 --- a/executorch-patches/qwen3_4b_decoder.patch +++ b/executorch-patches/qwen3_4b_decoder.patch @@ -1,5 +1,5 @@ diff --git a/examples/qualcomm/oss_scripts/llama/__init__.py b/examples/qualcomm/oss_scripts/llama/__init__.py -index 963db6e..953dc4c 100644 +index 963db6e..9ccfdd0 100644 --- a/examples/qualcomm/oss_scripts/llama/__init__.py +++ b/examples/qualcomm/oss_scripts/llama/__init__.py @@ -25,9 +25,14 @@ from executorch.examples.models.granite import ( @@ -20,7 +20,7 @@ index 963db6e..953dc4c 100644 from executorch.examples.models.qwen2_5 import ( convert_weights as convert_qwen2_5_weights, ) -@@ -479,6 +484,34 @@ class Qwen3_1_7B(LLMModelConfig): +@@ -479,6 +484,37 @@ class Qwen3_1_7B(LLMModelConfig): quant_recipe = Qwen3_1_7BQuantRecipe @@ -40,10 +40,13 @@ index 963db6e..953dc4c 100644 + convert_weights = convert_qwen3_weights + transform_weight = False + instruct_model = True -+ # Bumped to 2 to halve peak host RAM during QNN compile (4B at sharding=1 -+ # OOMed on a 62 GB box, peak anon-rss 46 GB). At sharding=2 each shard -+ # compile fits comfortably; runner stitches them at load time. -+ num_sharding = 2 ++ # num_sharding=1 for hybrid mode: sharding=2 produces a multi-context ++ # .pte (2 graphs × 2 shards = 4 contexts) that the LlmModule load path ++ # can't restore (error 5010 "Context group 1 does not exist"). With ++ # sharding=1 the hybrid export needs ~46 GB RAM peak — the 192 GB swap ++ # on /swapfile handles this; compile takes ~80 min wall but completes ++ # cleanly. Single-context .pte loads fine through the JNI runner. ++ num_sharding = 1 + masked_softmax = True + seq_mse_candidates = 0 + r1 = False