Update __init__.py

Ransaka · Feb 10, 2024 · 409c81c · 409c81c
1 parent 9195650
commit 409c81c
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/src/sinhala_tokenizers/__init__.py b/src/sinhala_tokenizers/__init__.py
@@ -0,0 +1,10 @@
+from sinlib.tokenizer import Tokenizer
+from sinlib.utils.preprocessing import remove_english_characters, remove_non_printable, retain_sinhala_characters, get_sinhala_token_percentage
+
+__all__ = [
+    "Tokenizer",
+    "remove_english_characters",
+    "remove_non_printable",
+    "retain_sinhala_characters",
+    "get_sinhala_token_percentage"
+]