added script to create tokenizers out of hf datasets

aya-multitokenizer · Jul 21, 2024 · d088808 · d088808
1 parent 02e2de7
commit d088808
Show file tree

Hide file tree

Showing 7 changed files with 122 additions and 108,894 deletions.
diff --git a/multi_tokenizer/pretrained/chinese_tokenizer.json b/multi_tokenizer/pretrained/chinese_tokenizer.json
@@ -49822,4 +49822,3 @@
       ]
     }
   }
-
diff --git a/multi_tokenizer/pretrained/spanish_tokenizer.json b/multi_tokenizer/pretrained/spanish_tokenizer.json
@@ -49822,4 +49822,3 @@
       ]
     }
   }
-
diff --git a/multi_tokenizer/tokenizer.py b/multi_tokenizer/tokenizer.py
@@ -44,7 +44,12 @@ def pre_tokenize(self, text: str) -> list[tuple[str, tuple[int, int]]]:
             output = (
                 [(tokenizer.language_prefix_token, (-1, 0))]
                 + output
-                + [(tokenizer.language_suffix_token, (len(detected_text) - 2, len(detected_text) - 1))]
+                + [
+                    (
+                        tokenizer.language_suffix_token,
+                        (len(detected_text) - 2, len(detected_text) - 1),
+                    )
+                ]
             )
             # Offsetting the start and end indices of the tokens to match the original text
             output = [