Engine sur fork qualcomm: decode dense 15.5>pte, 3.5 ~11, KVq8+t4+fa+lm_head-Q4; prefill 93/285; RAG cross 24-28k
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Richard Loyer
parent
1252f32bd3
commit
b2fc564f04
|
|
@ -4,3 +4,4 @@ models/
|
|||
*.pte
|
||||
*.log
|
||||
b/
|
||||
ql/
|
||||
|
|
|
|||
|
|
@ -8,14 +8,14 @@ Faire tourner Qwen3.5-4B (et 9B) en GGUF, sans conversion `.pte`, en exploitant
|
|||
## 2. Le fait dur (mesuré, reproductible)
|
||||
Decode M=1 = **memory-bandwidth bound**, pas compute. Le NPU n'aide pas le decode (bus DDR saturé), il aide le prefill (batché, compute-bound). Donc : **prefill NPU / decode CPU**, verrouillé pour 4B et 9B.
|
||||
|
||||
## 3. Performances stables (r3, ±2%) — moteur llama.cpp upstream
|
||||
| Modèle | prefill HTP | decode CPU | RAM |
|
||||
## 3. Performances stables — MAJ 26/05, fork ql (qualcomm/llama.cpp), t4+fa decode
|
||||
| Modèle | prefill (t8/b512) | decode (t4+fa) | RAM |
|
||||
|---|---:|---:|---:|
|
||||
| Qwen3.5-4B-Q4_0 | ~50 (ngl99/t8/b512) | 9.8 | 2.4 GB |
|
||||
| Qwen3.5-9B-Q4_0 | ~41 | 5.2 | 5.0 GB |
|
||||
| Qwen3-4B dense | 103 | 17 | 2.2 GB |
|
||||
| Qwen3.5-4B-Q4_0 | 93 | 8.5 | 2.4 GB |
|
||||
| Qwen3.5-9B-Q4_0 | 70 | 5.7 | 5.0 GB |
|
||||
| Qwen3-4B dense | 285 | 14 | 2.2 GB |
|
||||
| 4B ExecuTorch .pte | ~451 | 15 | 3.3 GB |
|
||||
Prefill 3.5 plafonne à 55 : les 24 couches GDN tournent CPU (pas de kernel HTP). Le dense fait 103 (matmuls HMX). 30B/35B = OOM. r1 antérieurs (8.4→2.1) = bruit, écartés.
|
||||
Decode dense 4B = parité pte à RAM −30%. NPU 9.82=CPU 9.8 → 77GB/s injoignable (GEMV M=1 latency-bound, ~32 réel). t4=optimum, t8=contention (3.5 chute 2.3). Fork qualcomm DOUBLE le prefill (50→93, 103→285) + sortie FR cohérente. Anciens 9.8/50/t8 obsolètes. Leviers decode morts (mesurés): spec-decode négatif, IQ4_NL=même octets, fusion GDN compute-bound. >9 decode = kernel GDN-HMX frontière.
|
||||
|
||||
## 4. Cascade Thinker+Guard : RAM OK
|
||||
9B+Guard-4B = 7.5 GB ; 9B+Guard-8B = 9.5 GB ; +TTS/STT → 11.5. Tient sur 16 GB. 9B>4B en qualité FR.
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ libllama.so, libggml{,-base,-cpu,-hexagon}.so, libggml-htp-v68/69/73/75/79/81.so
|
|||
Speaker Qwen3.5-9B-Q4_0 5.0GB (ou 4B 2.4). Thinker Guard-4B Q4_K_M 2.5GB. Cascade 7.5GB OK. tokenizer dans le GGUF.
|
||||
|
||||
## Perf cible (mesuré)
|
||||
9B prefill HTP 41 / decode CPU 8.4 ; 4B 55/15. 9B>4B qualité (Speaker=9B reco). cli load -ngl99 -dev HTP0 -t8. ENV: LD_LIBRARY_PATH+ADSP_LIBRARY_PATH=jniLibs.
|
||||
9B prefill 70 / decode 5.7 ; 3.5-4B 93/8.5 ; dense 4B 285/14. decode t4+fa (PAS t8), prefill t8/b512. 9B>4B qualité. cli -ngl99 -dev HTP0. ENV: LD_LIBRARY_PATH+ADSP_LIBRARY_PATH=jniLibs.
|
||||
|
||||
## Cascade (RAPPORT_KAZEIA prompts): Guard-4B bullets → 9B SYS_KAZEIA+bullets. reset() entre tours. JNI: kazeia_engine_jni.cpp + EngineLlmEngine.kt. TTS Talker/CP GGUF chargeables sur même backend (Talker prefill HTP).
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# Perf mesurée SM8750/V79 (prefill HTP ngl99/t8/b512, decode CPU t8)
|
||||
| Modèle | prefill | decode | RAM |
|
||||
|---|---|---|---|
|
||||
| Qwen3.5-4B | 55 | 15 | 2.4 |
|
||||
| Qwen3.5-9B | 41 | 8.4 | 5.0 |
|
||||
| Qwen3-4B dense | 103 | 17 | 2.2 |
|
||||
vs ExecuTorch 4B: 14-21 dec/3.3GB pte. Engine -30% RAM, decode equal. decode=BW plafond (~25GB/s). 9B max prefill via GDN HVX chunkwise (futur). cascade tour ~9s.
|
||||
# Perf mesurée SM8750/V79 — MAJ 26/05 (fork ql=qualcomm/llama.cpp)
|
||||
Decode optimum = **ngl99 / t4 / fa1** (PAS t8: contention, 3.5-4B chute à 2.3). Prefill optimum = t8 / b512 / fa1.
|
||||
| Modèle | prefill (t8/b512) | decode (t4+fa) | RAM |
|
||||
|---|---:|---:|---:|
|
||||
| Qwen3-4B dense | 285 | 14 (15.5 KVq8) | 2.2 |
|
||||
| Qwen3.5-4B | 93 | 8.5 (9.8 KVq8) | 2.4 |
|
||||
| Qwen3.5-9B | 70 | 5.7 | 5.0 |
|
||||
Dense 4B decode 14 = parité .pte (15) à RAM −30%. NPU 9.82=CPU 9.8: 77GB/s injoignable au decode (GEMV M=1 latency-bound, ~32 réel). Fork ql ~2x prefill vs ancien tree (50→93). RAM = GGUF mmap pageable + ~3.3GB ION/session NPU. Sortie FR cohérente prefill HTP. KV q8_0 (quasi-lossless) = +16% decode, dense 15.5 > pte, bridge l'active. Leviers morts: spec-decode net-négatif, ngram idem (M=k coûteux GDN), IQ4_NL=même octets, fusion GDN compute-bound. >10 decode 3.5 = kernel GDN-HMX (frontière). t8/9.8/50 obsolètes.
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_i
|
|||
// Utils
|
||||
// Create a buffer and allocate all the tensors in a ggml_context
|
||||
// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft
|
||||
// ggml_backend_alloc_ctx_tensors_from_buft returns NULL on failure or if all tensors in ctx are already allocated or zero-sized
|
||||
GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);
|
||||
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);
|
||||
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ extern "C" {
|
|||
GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
|
||||
|
||||
// tensor copy between different backends
|
||||
GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||
GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||
|
||||
//
|
||||
// Backend (stream)
|
||||
|
|
@ -83,12 +83,16 @@ extern "C" {
|
|||
GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);
|
||||
GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);
|
||||
|
||||
GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
|
||||
GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
|
||||
|
||||
// "offset" refers to the offset in tensor->data for setting/getting data
|
||||
GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
|
||||
GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
|
||||
GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
|
||||
|
||||
GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
|
||||
|
|
@ -109,7 +113,7 @@ extern "C" {
|
|||
// the copy is performed after all the currently queued operations in backend_src
|
||||
// backend_dst will wait for the copy to complete before performing other operations
|
||||
// automatic fallback to sync copy if async is not supported
|
||||
GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||
GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||
|
||||
GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);
|
||||
|
||||
|
|
@ -135,7 +139,9 @@ extern "C" {
|
|||
// integrated GPU device using host memory
|
||||
GGML_BACKEND_DEVICE_TYPE_IGPU,
|
||||
// accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)
|
||||
GGML_BACKEND_DEVICE_TYPE_ACCEL
|
||||
GGML_BACKEND_DEVICE_TYPE_ACCEL,
|
||||
// "meta" device wrapping multiple other devices for tensor parallelism
|
||||
GGML_BACKEND_DEVICE_TYPE_META,
|
||||
};
|
||||
|
||||
// functionality supported by the device
|
||||
|
|
@ -163,7 +169,7 @@ extern "C" {
|
|||
// device type
|
||||
enum ggml_backend_dev_type type;
|
||||
// device id
|
||||
// for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0")
|
||||
// for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0")
|
||||
// if the id is unknown, this should be NULL
|
||||
const char * device_id;
|
||||
// device capabilities
|
||||
|
|
@ -196,7 +202,12 @@ extern "C" {
|
|||
|
||||
// Common functions that may be obtained using ggml_backend_reg_get_proc_address
|
||||
|
||||
// Split buffer type for tensor parallelism
|
||||
// Context management and operations for faster communication between backends, used for tensor parallelism (meta backend)
|
||||
typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends);
|
||||
typedef void (*ggml_backend_comm_free_t)(void * comm_ctx);
|
||||
typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);
|
||||
|
||||
// Split buffer type for tensor parallelism (old)
|
||||
typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
|
||||
// Set the number of threads for the backend
|
||||
typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
|
||||
|
|
@ -340,6 +351,53 @@ extern "C" {
|
|||
// Set a callback to be called for each resulting node during graph compute
|
||||
GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
|
||||
|
||||
//
|
||||
// Meta backend
|
||||
//
|
||||
|
||||
#define GGML_BACKEND_META_MAX_DEVICES 16
|
||||
|
||||
enum ggml_backend_meta_split_axis {
|
||||
// tensor split by tensor dimensions:
|
||||
GGML_BACKEND_SPLIT_AXIS_0 = 0,
|
||||
GGML_BACKEND_SPLIT_AXIS_1 = 1,
|
||||
GGML_BACKEND_SPLIT_AXIS_2 = 2,
|
||||
GGML_BACKEND_SPLIT_AXIS_3 = 3,
|
||||
|
||||
GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends
|
||||
GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum
|
||||
|
||||
// for internal bookkeeping only:
|
||||
GGML_BACKEND_SPLIT_AXIS_NONE = 98,
|
||||
GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99,
|
||||
};
|
||||
GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis);
|
||||
|
||||
struct ggml_backend_meta_split_state {
|
||||
enum ggml_backend_meta_split_axis axis;
|
||||
|
||||
// for tensors with axis >= 0 && axis < GGML_MAX_DIMS:
|
||||
// - each device has a slice of the tensor along the split axis
|
||||
// - most tensors have n_segments == 1 and a contiguous slice of the tensor data
|
||||
// - some tensors have an inhomogenenous data layout along the split axis,
|
||||
// those tensors are divided into segments which are each individually split across devices
|
||||
// - ne has one entry per segment and device that add up to ggml_tensor::ne for that axis,
|
||||
// the outer/inner loops are over segments/devices like [seg0_dev0, seg0_dev1, seg1_dev0, seg1_dev1],
|
||||
// - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments
|
||||
// that each need to be split individually across devices so that each device gets a slice of Q, K, and V
|
||||
int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES];
|
||||
uint32_t n_segments;
|
||||
};
|
||||
|
||||
// function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible:
|
||||
typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata);
|
||||
|
||||
// create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this:
|
||||
// TODO: this looks a bit strange - a backend API creates a device. I think we should try
|
||||
// express this as a backend registry functionality instead
|
||||
GGML_API ggml_backend_dev_t ggml_backend_meta_device(
|
||||
ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud);
|
||||
|
||||
//
|
||||
// Utils
|
||||
//
|
||||
|
|
|
|||
|
|
@ -27,6 +27,9 @@ GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);
|
|||
// device buffer
|
||||
GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
|
||||
|
||||
// conduct allreduce operation between devices
|
||||
GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends);
|
||||
|
||||
// split tensor buffer that splits matrices by rows across multiple devices
|
||||
GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split);
|
||||
|
||||
|
|
|
|||
|
|
@ -14,49 +14,6 @@ GGML_BACKEND_API bool ggml_backend_is_hexagon(ggml_backend_t backend);
|
|||
|
||||
GGML_BACKEND_API ggml_backend_reg_t ggml_backend_hexagon_reg(void);
|
||||
|
||||
// Mega-kernel transport self-test. Allocates 3 small FastRPC buffers
|
||||
// (descriptor, ack, logits), ships a minimal-but-valid htp_mega_descriptor
|
||||
// to the DSP via HTP_OP_QWEN35_COMPILE, invokes HTP_OP_QWEN35_STEP, reads
|
||||
// back the stub-produced logits (should be all zeros), and verifies.
|
||||
//
|
||||
// Returns 0 on success, negative on failure. Logs diagnostic to stderr.
|
||||
// Scope: proves the host↔DSP transport for the mega-kernel entry points
|
||||
// with the skeleton stubs from qwen35-forward-ops.c. No real inference.
|
||||
GGML_BACKEND_API int ggml_backend_hexagon_qwen35_selftest(ggml_backend_t backend);
|
||||
|
||||
// Mega-kernel graph compiler. Walks the given ggml_cgraph, emits an
|
||||
// htp_mega_descriptor (ops + tensor table + buffer table), registers every
|
||||
// distinct ggml_backend_buffer_t the graph touches as a persistent FastRPC
|
||||
// buffer, ships everything to the DSP in a single HTP_OP_QWEN35_COMPILE
|
||||
// message, and stores the descriptor in the DSP's static slot.
|
||||
//
|
||||
// Tensor table entries carry (buffer_id, offset_in_bytes) so the DSP
|
||||
// interpreter can resolve any tensor id to a pointer in O(1). Op entries
|
||||
// reference src/dst tensor ids; unsupported GGML ops are emitted as
|
||||
// HTP_MEGA_NOP (the interpreter will skip them).
|
||||
//
|
||||
// logits_tensor identifies which tensor's data should be copied back to the
|
||||
// host when a subsequent step call completes. For a real Qwen3.5 model this
|
||||
// is the lm_head output tensor; for a test graph it can be any intermediate
|
||||
// node. Pass NULL to default to the last cgraph node.
|
||||
//
|
||||
// Returns 0 on success, negative on failure.
|
||||
GGML_BACKEND_API int ggml_backend_hexagon_qwen35_compile(
|
||||
ggml_backend_t backend,
|
||||
struct ggml_cgraph * cgraph,
|
||||
struct ggml_tensor * logits_tensor);
|
||||
|
||||
// Invokes one forward-pass step of the compiled mega-kernel on the DSP.
|
||||
// The descriptor must have been shipped via ggml_backend_hexagon_qwen35_compile.
|
||||
// input_token_id + position identify the current decode input; logits_out
|
||||
// receives n_logits floats (must be ≥ descriptor vocab_size).
|
||||
GGML_BACKEND_API int ggml_backend_hexagon_qwen35_step(
|
||||
ggml_backend_t backend,
|
||||
uint32_t input_token_id,
|
||||
int32_t position,
|
||||
float * logits_out,
|
||||
size_t n_logits);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define RPC_PROTO_MAJOR_VERSION 3
|
||||
#define RPC_PROTO_MINOR_VERSION 6
|
||||
#define RPC_PROTO_PATCH_VERSION 1
|
||||
#define RPC_PROTO_MAJOR_VERSION 4
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 0
|
||||
|
||||
#ifdef __cplusplus
|
||||
static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
|
|
|
|||
|
|
@ -428,7 +428,8 @@ extern "C" {
|
|||
// GGML_TYPE_IQ4_NL_8_8 = 38,
|
||||
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
|
||||
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
|
||||
GGML_TYPE_COUNT = 41,
|
||||
GGML_TYPE_Q1_0 = 41,
|
||||
GGML_TYPE_COUNT = 42,
|
||||
};
|
||||
|
||||
// precision
|
||||
|
|
@ -437,6 +438,12 @@ extern "C" {
|
|||
GGML_PREC_F32 = 10,
|
||||
};
|
||||
|
||||
// op hint
|
||||
enum ggml_op_hint {
|
||||
GGML_HINT_NONE = 0,
|
||||
GGML_HINT_SRC0_IS_HADAMARD = 1,
|
||||
};
|
||||
|
||||
// model file types
|
||||
enum ggml_ftype {
|
||||
GGML_FTYPE_UNKNOWN = -1,
|
||||
|
|
@ -465,6 +472,7 @@ extern "C" {
|
|||
GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
|
||||
GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors
|
||||
GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors
|
||||
GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors
|
||||
};
|
||||
|
||||
// available tensor operations:
|
||||
|
|
@ -900,15 +908,17 @@ extern "C" {
|
|||
struct ggml_tensor * b,
|
||||
struct ggml_tensor * ids);
|
||||
|
||||
GGML_API struct ggml_tensor * ggml_add1(
|
||||
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
struct ggml_tensor * b);
|
||||
struct ggml_tensor * b),
|
||||
"use ggml_add instead");
|
||||
|
||||
GGML_API struct ggml_tensor * ggml_add1_inplace(
|
||||
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
struct ggml_tensor * b);
|
||||
struct ggml_tensor * b),
|
||||
"use ggml_add_inplace instead");
|
||||
|
||||
// dst = a
|
||||
// view(dst, nb1, nb2, nb3, offset) += b
|
||||
|
|
@ -1179,8 +1189,8 @@ extern "C" {
|
|||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a);
|
||||
|
||||
// a - x
|
||||
// b - dy
|
||||
// a - dy
|
||||
// b - x
|
||||
GGML_API struct ggml_tensor * ggml_silu_back(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
|
|
@ -1415,6 +1425,11 @@ extern "C" {
|
|||
struct ggml_tensor * a,
|
||||
enum ggml_prec prec);
|
||||
|
||||
// change the hint of a matrix multiplication
|
||||
GGML_API void ggml_mul_mat_set_hint(
|
||||
struct ggml_tensor * a,
|
||||
enum ggml_op_hint hint);
|
||||
|
||||
// indirect matrix multiplication
|
||||
GGML_API struct ggml_tensor * ggml_mul_mat_id(
|
||||
struct ggml_context * ctx,
|
||||
|
|
@ -1769,8 +1784,32 @@ extern "C" {
|
|||
int n_dims,
|
||||
int mode);
|
||||
|
||||
// custom RoPE
|
||||
// RoPE operations with extended options
|
||||
// a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token]
|
||||
// b is an int32 vector with size n_token
|
||||
// c is freq factors (e.g. phi3-128k), (optional)
|
||||
// mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi
|
||||
//
|
||||
// pseudo-code for computing theta:
|
||||
// for i in [0, n_dims/2):
|
||||
// theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims);
|
||||
// theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c
|
||||
// theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here
|
||||
//
|
||||
// other params are used by YaRN RoPE scaling, these default values will disable YaRN:
|
||||
// freq_scale = 1.0f
|
||||
// ext_factor = 0.0f
|
||||
// attn_factor = 1.0f
|
||||
// beta_fast = 0.0f
|
||||
// beta_slow = 0.0f
|
||||
//
|
||||
// example:
|
||||
// (marking: c = cos, s = sin, 0 = unrotated)
|
||||
// given a single head with size = 8 --> [00000000]
|
||||
// GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000]
|
||||
// GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs]
|
||||
// GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000]
|
||||
// GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss]
|
||||
GGML_API struct ggml_tensor * ggml_rope_ext(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
|
|
@ -1786,6 +1825,36 @@ extern "C" {
|
|||
float beta_fast,
|
||||
float beta_slow);
|
||||
|
||||
// multi-dimensional RoPE, for Qwen-VL and similar vision models
|
||||
// mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX
|
||||
// sections specify how many dimensions to rotate in each section:
|
||||
// section length is equivalent to number of cos/sin pairs, NOT the number of dims
|
||||
// (i.e. sum of 4 sections are expected to be n_dims/2)
|
||||
// last sections can be 0, means ignored
|
||||
// all other options are identical to ggml_rope_ext
|
||||
//
|
||||
// important note:
|
||||
// - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION
|
||||
// if you need normal ordering, there are 2 methods:
|
||||
// (1) split the tensor manually using ggml_view
|
||||
// (2) permute the weight upon conversion
|
||||
// - for VISION, n_dims must be head_size/2
|
||||
//
|
||||
// example M-RoPE:
|
||||
// given sections = [t=4, y=2, x=2, 0]
|
||||
// given a single head with size = 18 --> [000000000000000000]
|
||||
// GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering)
|
||||
// GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering)
|
||||
// note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section
|
||||
// in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section
|
||||
//
|
||||
// example vision RoPE:
|
||||
// given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored)
|
||||
// given a single head with size = 8 --> [00000000]
|
||||
// GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx]
|
||||
// other values of n_dims are untested and is undefined behavior
|
||||
// note: unlike MROPE, the theta for each dim is computed differently for each section
|
||||
// in other words, idx used for theta: [0123] for y section, then [0123] for x section
|
||||
GGML_API struct ggml_tensor * ggml_rope_multi(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
|
|
@ -2472,6 +2541,11 @@ extern "C" {
|
|||
|
||||
// TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST]
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306
|
||||
//
|
||||
// state is a 3D tensor of shape (S_v*S_v*H, K, n_seqs):
|
||||
// K == 1: output carries the final state only.
|
||||
// K > 1: output carries K snapshot slots; the kernel writes the last min(n_tokens, K)
|
||||
// per-token snapshots into the trailing slots
|
||||
GGML_API struct ggml_tensor * ggml_gated_delta_net(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * q,
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ extern "C" {
|
|||
LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors
|
||||
LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors
|
||||
LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors
|
||||
LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors
|
||||
|
||||
LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
|
||||
};
|
||||
|
|
@ -194,6 +195,12 @@ extern "C" {
|
|||
LLAMA_SPLIT_MODE_NONE = 0, // single GPU
|
||||
LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs
|
||||
LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported
|
||||
LLAMA_SPLIT_MODE_TENSOR = 3,
|
||||
};
|
||||
|
||||
enum llama_context_type {
|
||||
LLAMA_CONTEXT_TYPE_DEFAULT = 0,
|
||||
LLAMA_CONTEXT_TYPE_MTP = 1,
|
||||
};
|
||||
|
||||
// TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)
|
||||
|
|
@ -331,9 +338,11 @@ extern "C" {
|
|||
uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode
|
||||
uint32_t n_ubatch; // physical maximum batch size
|
||||
uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)
|
||||
uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
|
||||
int32_t n_threads; // number of threads to use for generation
|
||||
int32_t n_threads_batch; // number of threads to use for batch processing
|
||||
|
||||
enum llama_context_type ctx_type; // set the context type (e.g. MTP)
|
||||
enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`
|
||||
enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id
|
||||
enum llama_attention_type attention_type; // attention type to use for embeddings
|
||||
|
|
@ -380,6 +389,17 @@ extern "C" {
|
|||
size_t n_samplers;
|
||||
};
|
||||
|
||||
struct llama_model_tensor_override {
|
||||
const char * pattern;
|
||||
enum ggml_type type;
|
||||
};
|
||||
|
||||
struct llama_model_imatrix_data {
|
||||
const char * name;
|
||||
const float * data;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
// model quantization parameters
|
||||
typedef struct llama_model_quantize_params {
|
||||
int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
|
||||
|
|
@ -392,10 +412,10 @@ extern "C" {
|
|||
bool pure; // quantize all tensors to the default type
|
||||
bool keep_split; // quantize to the same number of shards
|
||||
bool dry_run; // calculate and show the final quantization size without performing quantization
|
||||
void * imatrix; // pointer to importance matrix data
|
||||
void * kv_overrides; // pointer to vector containing overrides
|
||||
void * tensor_types; // pointer to vector containing tensor types
|
||||
void * prune_layers; // pointer to vector containing layer indices to prune
|
||||
const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data
|
||||
const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides
|
||||
const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides
|
||||
const int32_t * prune_layers; // pointer to layer indices to prune
|
||||
} llama_model_quantize_params;
|
||||
|
||||
typedef struct llama_logit_bias {
|
||||
|
|
@ -465,6 +485,11 @@ extern "C" {
|
|||
const char * path_model,
|
||||
struct llama_model_params params);
|
||||
|
||||
// Load a model from an open FILE pointer
|
||||
LLAMA_API struct llama_model * llama_model_load_from_file_ptr(
|
||||
FILE * file,
|
||||
struct llama_model_params params);
|
||||
|
||||
// Load a model from multiple splits (support custom naming scheme)
|
||||
// The paths must be in the correct order
|
||||
LLAMA_API struct llama_model * llama_model_load_from_splits(
|
||||
|
|
@ -493,27 +518,6 @@ extern "C" {
|
|||
// Frees all allocated memory
|
||||
LLAMA_API void llama_free(struct llama_context * ctx);
|
||||
|
||||
enum llama_params_fit_status {
|
||||
LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit
|
||||
LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit
|
||||
LLAMA_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path
|
||||
};
|
||||
|
||||
// fits mparams and cparams to free device memory (assumes system memory is unlimited)
|
||||
// - returns true if the parameters could be successfully modified to fit device memory
|
||||
// - this function is NOT thread safe because it modifies the global llama logger state
|
||||
// - only parameters that have the same value as in llama_default_model_params are modified
|
||||
// with the exception of the context size which is modified if and only if equal to 0
|
||||
LLAMA_API enum llama_params_fit_status llama_params_fit(
|
||||
const char * path_model,
|
||||
struct llama_model_params * mparams,
|
||||
struct llama_context_params * cparams,
|
||||
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
|
||||
struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
|
||||
size_t * margins, // margins of memory to leave per device in bytes
|
||||
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
|
||||
enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
|
||||
|
||||
LLAMA_API int64_t llama_time_us(void);
|
||||
|
||||
LLAMA_API size_t llama_max_devices(void);
|
||||
|
|
@ -533,6 +537,7 @@ extern "C" {
|
|||
LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);
|
||||
LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);
|
||||
LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);
|
||||
LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx);
|
||||
|
||||
DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");
|
||||
DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead");
|
||||
|
|
@ -861,12 +866,18 @@ extern "C" {
|
|||
size_t n_token_capacity,
|
||||
size_t * n_token_count_out);
|
||||
|
||||
#define LLAMA_STATE_SEQ_FLAGS_NONE 0
|
||||
|
||||
// for backwards-compat
|
||||
#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1
|
||||
|
||||
// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)
|
||||
#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1
|
||||
|
||||
// Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load).
|
||||
// Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag.
|
||||
#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2
|
||||
|
||||
typedef uint32_t llama_state_seq_flags;
|
||||
|
||||
LLAMA_API size_t llama_state_seq_get_size_ext(
|
||||
|
|
@ -1528,9 +1539,6 @@ extern "C" {
|
|||
LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain);
|
||||
LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain);
|
||||
|
||||
// print a breakdown of per-device memory use via LLAMA_LOG:
|
||||
LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx);
|
||||
|
||||
//
|
||||
// training
|
||||
//
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// kazeia_engine_jni.cpp — bridge LLM Kazeia-Engine (llama.cpp upstream + Hexagon).
|
||||
// Speaker+Thinker, GGUF, prefill NPU (ngl99/HTP0) + decode CPU (8 threads).
|
||||
// Speaker+Thinker, GGUF, prefill NPU (ngl99/HTP0) + decode CPU (t4+fa = 14t/s; t8=contention).
|
||||
// thinking off via template applique cote Kotlin (/no_think) OU enable_thinking=false.
|
||||
// STT reste ORT-QAIRT. Build: link libllama+libcommon+ggml*; jniLibs htp auto-V79.
|
||||
#include <jni.h>
|
||||
|
|
@ -16,7 +16,9 @@ Java_com_kazeia_llm_EngineJni_load(JNIEnv* e, jobject, jstring path, jint nctx)
|
|||
auto mp = llama_model_default_params(); mp.n_gpu_layers = 0; // decode CPU pur 14t/s (ngl99=ping-pong 0.2)
|
||||
auto m = llama_model_load_from_file(p, mp); e->ReleaseStringUTFChars(path, p);
|
||||
if (!m) return 0;
|
||||
auto cp = llama_context_default_params(); cp.n_ctx = nctx; cp.n_threads = 8; cp.n_batch = 512;
|
||||
auto cp = llama_context_default_params(); cp.n_ctx = nctx; cp.n_threads = 4; cp.n_batch = 512;
|
||||
cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; // t4+fa = 14t/s decode (t8=9.8 contention; fa coupe trafic KV)
|
||||
cp.type_k = GGML_TYPE_Q8_0; cp.type_v = GGML_TYPE_Q8_0; // KV q8: 14->15.5 dense (>pte), 8.5->9.8 hyb, quasi-lossless
|
||||
auto* k = new KEngine{m, llama_init_from_model(m, cp), llama_model_get_vocab(m),
|
||||
llama_sampler_init_greedy()};
|
||||
return (jlong)k;
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue