// Test bit-exact du tokenizer C++ vs input_ids_full.bin (golden HF tokenizer).
// Usage : test_tokenizer <vocab_gguf> <golden_input_ids.bin> "<content>"
//   ex:    test_tokenizer Qwen3-4B-Q4_0.gguf tts_dump/input_ids_full.bin "Bonjour je m'appelle Kazeia"
//
// Critère : tokens C++ == tokens golden (byte-pour-byte).
#include "kazeia_text_tokenizer.h"
#include "llama.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <vector>

int main(int argc, char ** argv) {
    if (argc < 4) {
        fprintf(stderr, "usage: %s <vocab_gguf> <golden_input_ids.bin> \"<content>\"\n", argv[0]);
        return 1;
    }
    const char * vocab_path  = argv[1];
    const char * golden_path = argv[2];
    const std::string content = argv[3];

    llama_backend_init();

    KzTextTokenizer tok;
    if (!kz_tok_load(tok, vocab_path)) return 2;

    auto ids = kz_tok_encode_tts_prompt(tok, content);
    printf("ids (%zu):", ids.size());
    for (auto i : ids) printf(" %d", i);
    printf("\n");

    // Lire golden
    std::ifstream f(golden_path, std::ios::binary | std::ios::ate);
    if (!f) { fprintf(stderr, "cannot open golden %s\n", golden_path); return 3; }
    size_t n_bytes = (size_t)f.tellg();
    f.seekg(0);
    std::vector<int32_t> golden(n_bytes / sizeof(int32_t));
    f.read((char*)golden.data(), n_bytes);
    printf("golden (%zu):", golden.size());
    for (auto i : golden) printf(" %d", i);
    printf("\n");

    bool ok = (ids.size() == golden.size());
    if (ok) {
        for (size_t i = 0; i < ids.size(); ++i) {
            if (ids[i] != golden[i]) { ok = false; printf("  diff at %zu: got %d, want %d\n", i, ids[i], golden[i]); }
        }
    } else {
        printf("  size mismatch: got %zu, want %zu\n", ids.size(), golden.size());
    }
    printf(ok ? "OK BIT-EXACT\n" : "MISMATCH\n");

    kz_tok_free(tok);
    llama_backend_free();
    return ok ? 0 : 4;
}