Add tokenizer tests

huggingface · Nov 21, 2023 · 7b7608f · 7b7608f
1 parent 7dfaceb
commit 7b7608f
Show file tree

Hide file tree

Showing 2 changed files with 564 additions and 0 deletions.
diff --git a/src/transformers/models/siglip/tokenization_siglip.py b/src/transformers/models/siglip/tokenization_siglip.py
@@ -121,6 +121,7 @@ def __init__(
         split_special_tokens=True,
         **kwargs,
     ) -> None:
+        print("Pad token:", pad_token)
         pad_token = AddedToken(pad_token, rstrip=True, lstrip=True)
         unk_token = AddedToken(unk_token, rstrip=True, lstrip=True)
         eos_token = AddedToken(eos_token, rstrip=True, lstrip=True)