Bug fixing

Ransaka · Sep 3, 2024 · 4d4aa76 · 4d4aa76
1 parent a9b2d80
commit 4d4aa76
Show file tree

Hide file tree

Showing 4 changed files with 1,028 additions and 2,282 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "sinlib"
-version = "0.1.4"
+version = "0.1.5"
 description = "Sinhala NLP Toolkit"
 authors = [
     { name = "Ransaka", email = "ransaka.ravihara@gmail.com" }

diff --git a/src/sinlib/tokenizer.py b/src/sinlib/tokenizer.py
@@ -58,7 +58,8 @@ def __encode(self, text, truncate_and_pad: bool, allowed_special_tokens: list =
                         text_encodings.append(self.vocab_map[token])
                     else:
                         continue
-
+                else:
+                    text_encodings.append(self.vocab_map.get(token, self.unknown_token_id))
         if truncate_and_pad:
             return self.pad_or_truncate(
                 sequence=text_encodings,

diff --git a/src/sinlib/utils/data/config.json b/src/sinlib/utils/data/config.json
@@ -1,7 +1,9 @@
 {
-    "unknown_token": "<unk>",
-    "pad_token": "<pad>",
-    "unknown_token_id": 126,
-    "pad_token_id": 127,
-    "max_length": 30
+    "unknown_token": "<|unk|>",
+    "pad_token": "<|pad|>",
+    "unknown_token_id": 1015,
+    "pad_token_id": 1016,
+    "max_length": 256,
+    "end_of_text_token": "<|endoftext|>",
+    "end_of_text_token_id": 1017
 }