66 lines
2.8 KiB
Bash
Executable File
66 lines
2.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Éval qualité Kazeia : A/B aveugle entre deux GGUF, decode CPU + template + reasoning off.
|
|
# Génère un cahier de notation aveugle (results_blind.md) + la clé (results_key.csv).
|
|
# La notation est HUMAINE (voir PROTOCOLE_QUALITE.md). Pas de juge automatique.
|
|
#
|
|
# Usage: ./run_quality_eval.sh [modelA.gguf labelA] [modelB.gguf labelB]
|
|
# Défaut: q35-lmq4.gguf "Qwen3.5-4B-lmq4" vs Qwen3-4B-Q4_0.gguf "Qwen3-4B-dense"
|
|
set -euo pipefail
|
|
|
|
DDIR=/data/local/tmp/kz-engine
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
OUT="$HERE/out"; RAW="$OUT/raw"
|
|
NTOK=${NTOK:-150}
|
|
|
|
MA=${1:-q35-lmq4.gguf}; LA=${2:-Qwen3.5-4B-lmq4}
|
|
MB=${3:-Qwen3-4B-Q4_0.gguf}; LB=${4:-Qwen3-4B-dense}
|
|
|
|
mkdir -p "$RAW"
|
|
adb get-state >/dev/null 2>&1 || { echo "ERREUR: aucun device adb (rebrancher/déverrouiller le Pad3)." >&2; exit 1; }
|
|
adb push "$HERE/system_fr.txt" "$DDIR/_sys.txt" >/dev/null
|
|
|
|
gen() { # $1=model $2=device-prompt-file -> stdout = réponse nettoyée
|
|
adb shell "cd $DDIR; LD_LIBRARY_PATH=./lib ADSP_LIBRARY_PATH=./lib ./llama-cli -m $1 \
|
|
-dev none -t 4 -fa 1 -ctk q8_0 -ctv q8_0 --reasoning off --reasoning-budget 0 \
|
|
-cnv -st --temp 0 -n $NTOK -sysf _sys.txt -f $2 2>/dev/null" </dev/null \
|
|
| tr -d '\r' | awk 'f; /^> /{f=1}' | sed '/\[ Prompt:/,$d' \
|
|
| sed -E 's#^[[:cntrl:][:space:]|/\\-]+##' | sed '/^[[:space:]]*$/d'
|
|
}
|
|
|
|
BLIND="$OUT/results_blind.md"; KEY="$OUT/results_key.csv"
|
|
echo "# Éval qualité Kazeia — notation aveugle ($(date +%F))" > "$BLIND"
|
|
echo "" >> "$BLIND"
|
|
echo "Modèles comparés (ordre masqué par item) : **$LA** vs **$LB**. Decode CPU t4+fa, KV q8_0, reasoning off, temp 0." >> "$BLIND"
|
|
echo "Pour chaque item, note la meilleure réponse : \`1\`, \`2\`, ou \`=\` (égalité). Voir PROTOCOLE_QUALITE.md." >> "$BLIND"
|
|
echo "" >> "$BLIND"
|
|
echo "idx,categorie,position_1,position_2" > "$KEY"
|
|
|
|
i=0
|
|
while IFS= read -r line <&3; do
|
|
case "$line" in ''|\#*) continue;; esac
|
|
cat="${line%%|*}"; prompt="${line#*|}"
|
|
i=$((i+1))
|
|
printf '%s' "$prompt" > /tmp/_kz_p.txt
|
|
adb push /tmp/_kz_p.txt "$DDIR/_p.txt" >/dev/null
|
|
echo "[$i] $cat : $prompt" >&2
|
|
ra="$(gen "$MA" _p.txt || true)"; printf '%s\n' "$ra" > "$RAW/$(printf '%02d' $i)_A_${LA}.txt"
|
|
rb="$(gen "$MB" _p.txt || true)"; printf '%s\n' "$rb" > "$RAW/$(printf '%02d' $i)_B_${LB}.txt"
|
|
if [ $((RANDOM % 2)) -eq 0 ]; then p1="$ra"; p2="$rb"; m1="$LA"; m2="$LB"; else p1="$rb"; p2="$ra"; m1="$LB"; m2="$LA"; fi
|
|
echo "$i,$cat,$m1,$m2" >> "$KEY"
|
|
{
|
|
echo "## Item $i — [$cat]"
|
|
echo "**Demande :** $prompt"
|
|
echo ""
|
|
echo "**Réponse 1 :** $p1"
|
|
echo ""
|
|
echo "**Réponse 2 :** $p2"
|
|
echo ""
|
|
echo "_Meilleure (1 / 2 / =) : ___ Note 1 (/5) : ___ Note 2 (/5) : ___ Remarque : ___________________"
|
|
echo ""
|
|
echo "---"
|
|
echo ""
|
|
} >> "$BLIND"
|
|
done 3< "$HERE/prompts_fr.txt"
|
|
|
|
echo "OK -> $BLIND ($i items) | clé: $KEY" >&2
|