64 lines
2.9 KiB
Diff
64 lines
2.9 KiB
Diff
diff --git a/conversion/bert.py b/conversion/bert.py
|
|
index 8af6c53..6dfe463 100644
|
|
--- a/conversion/bert.py
|
|
+++ b/conversion/bert.py
|
|
@@ -28,6 +28,16 @@ class BertModel(TextModel):
|
|
cls_out_labels = None
|
|
self.cls_out_labels = cls_out_labels
|
|
|
|
+ # Certains modèles déclarent architectures=["BertModel"] mais utilisent un
|
|
+ # tokenizer XLM-RoBERTa SentencePiece/Unigram (ex. intfloat/multilingual-e5-*,
|
|
+ # CamemBERT). Détecter ici pour router le vocab + chopper la matrice de positions
|
|
+ # exactement comme RobertaModel/NomicBertModel le font déjà.
|
|
+ self._tokenizer_is_xlmroberta = self._is_tokenizer_xlmroberta()
|
|
+ if self._tokenizer_is_xlmroberta:
|
|
+ self._xlmroberta_tokenizer_init()
|
|
+ else:
|
|
+ self._position_offset = None
|
|
+
|
|
def set_gguf_parameters(self):
|
|
super().set_gguf_parameters()
|
|
self.gguf_writer.add_causal_attention(False)
|
|
@@ -37,6 +47,8 @@ class BertModel(TextModel):
|
|
self.gguf_writer.add_classifier_output_labels([v for k, v in sorted(self.cls_out_labels.items())])
|
|
|
|
def set_vocab(self):
|
|
+ if getattr(self, "_tokenizer_is_xlmroberta", False):
|
|
+ return self._xlmroberta_set_vocab()
|
|
tokens, toktypes, tokpre = self.get_vocab_base()
|
|
self.vocab_size = len(tokens)
|
|
|
|
@@ -91,6 +103,11 @@ class BertModel(TextModel):
|
|
return super().filter_tensors((name, gen))
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
+ # XLM-R : chopper la matrice de positions de _position_offset (pad/bos),
|
|
+ # comme RobertaModel/NeoBert. No-op si _position_offset is None (BERT WordPiece).
|
|
+ if name == "embeddings.position_embeddings.weight" and getattr(self, "_position_offset", None) is not None:
|
|
+ data_torch = data_torch[self._position_offset:, :]
|
|
+
|
|
if self.cls_out_labels:
|
|
# For BertForSequenceClassification (direct projection layer)
|
|
if name == "classifier.weight":
|
|
@@ -101,6 +118,20 @@ class BertModel(TextModel):
|
|
|
|
yield from super().modify_tensors(data_torch, name, bid)
|
|
|
|
+ # Détection robuste du tokenizer XLM-RoBERTa (SentencePiece/Unigram).
|
|
+ # Présent aussi sur NomicBertModel ; remonté en base pour que tout modèle
|
|
+ # déclaré "BertModel" mais à tokenizer Unigram (e5 multilingue, CamemBERT) marche.
|
|
+ def _is_tokenizer_xlmroberta(self) -> bool:
|
|
+ tj = self.dir_model / "tokenizer.json"
|
|
+ if not tj.is_file():
|
|
+ return False
|
|
+ try:
|
|
+ with open(tj) as f:
|
|
+ toktyp = json.load(f)["model"]["type"]
|
|
+ except Exception:
|
|
+ return False
|
|
+ return toktyp == "Unigram"
|
|
+
|
|
def _xlmroberta_tokenizer_init(self) -> None:
|
|
# we need the pad_token_id to know how to chop down position_embd matrix
|
|
if (pad_token_id := self.hparams.get("pad_token_id")) is not None:
|