Used wrong default clean fn in SimpleTokenizer, put lower case back

mlfoundations · Oct 12, 2023 · 2c396d2 · 2c396d2
1 parent 2f568cd
commit 2c396d2
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/open_clip/tokenizer.py b/src/open_clip/tokenizer.py
@@ -143,7 +143,7 @@ def __init__(
         if canonicalize:
             self.clean_fn = _canonicalize_basic_clean
         else:
-            self.clean_fn = _whitespace_basic_clean
+            self.clean_fn = _lower_whitespace_basic_clean
         self.vocab_size = len(self.encoder)
         self.all_special_ids = [self.encoder[t] for t in special_tokens]