Set normalized=False for special tokens

huggingface · Jan 2, 2024 · ffdcfc5 · ffdcfc5
1 parent 985a922
commit ffdcfc5
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/transformers/models/siglip/tokenization_siglip.py b/src/transformers/models/siglip/tokenization_siglip.py
@@ -116,12 +116,12 @@ def __init__(
             else pad_token
         )
         unk_token = (
-            AddedToken(unk_token, rstrip=True, lstrip=True, normalized=True, special=True)
+            AddedToken(unk_token, rstrip=True, lstrip=True, normalized=False, special=True)
             if isinstance(unk_token, str)
             else unk_token
         )
         eos_token = (
-            AddedToken(eos_token, rstrip=True, lstrip=True, normalized=True, special=True)
+            AddedToken(eos_token, rstrip=True, lstrip=True, normalized=False, special=True)
             if isinstance(eos_token, str)
             else eos_token
         )