58 lines
1.9 KiB
C++
58 lines
1.9 KiB
C++
// Test bit-exact du tokenizer C++ vs input_ids_full.bin (golden HF tokenizer).
|
|
// Usage : test_tokenizer <vocab_gguf> <golden_input_ids.bin> "<content>"
|
|
// ex: test_tokenizer Qwen3-4B-Q4_0.gguf tts_dump/input_ids_full.bin "Bonjour je m'appelle Kazeia"
|
|
//
|
|
// Critère : tokens C++ == tokens golden (byte-pour-byte).
|
|
#include "kazeia_text_tokenizer.h"
|
|
#include "llama.h"
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <fstream>
|
|
#include <vector>
|
|
|
|
int main(int argc, char ** argv) {
|
|
if (argc < 4) {
|
|
fprintf(stderr, "usage: %s <vocab_gguf> <golden_input_ids.bin> \"<content>\"\n", argv[0]);
|
|
return 1;
|
|
}
|
|
const char * vocab_path = argv[1];
|
|
const char * golden_path = argv[2];
|
|
const std::string content = argv[3];
|
|
|
|
llama_backend_init();
|
|
|
|
KzTextTokenizer tok;
|
|
if (!kz_tok_load(tok, vocab_path)) return 2;
|
|
|
|
auto ids = kz_tok_encode_tts_prompt(tok, content);
|
|
printf("ids (%zu):", ids.size());
|
|
for (auto i : ids) printf(" %d", i);
|
|
printf("\n");
|
|
|
|
// Lire golden
|
|
std::ifstream f(golden_path, std::ios::binary | std::ios::ate);
|
|
if (!f) { fprintf(stderr, "cannot open golden %s\n", golden_path); return 3; }
|
|
size_t n_bytes = (size_t)f.tellg();
|
|
f.seekg(0);
|
|
std::vector<int32_t> golden(n_bytes / sizeof(int32_t));
|
|
f.read((char*)golden.data(), n_bytes);
|
|
printf("golden (%zu):", golden.size());
|
|
for (auto i : golden) printf(" %d", i);
|
|
printf("\n");
|
|
|
|
bool ok = (ids.size() == golden.size());
|
|
if (ok) {
|
|
for (size_t i = 0; i < ids.size(); ++i) {
|
|
if (ids[i] != golden[i]) { ok = false; printf(" diff at %zu: got %d, want %d\n", i, ids[i], golden[i]); }
|
|
}
|
|
} else {
|
|
printf(" size mismatch: got %zu, want %zu\n", ids.size(), golden.size());
|
|
}
|
|
printf(ok ? "OK BIT-EXACT\n" : "MISMATCH\n");
|
|
|
|
kz_tok_free(tok);
|
|
llama_backend_free();
|
|
return ok ? 0 : 4;
|
|
}
|