cantokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tokenizers import Tokenizer, AddedToken, pre_tokenizers, decoders, trainers\n",
    "from tokenizers.models import WordPiece\n",
    "from tokenizers.normalizers import NFKC, Lowercase, Sequence\n",
    "\n",
    "from tokenizers.normalizers import BertNormalizer\n",
    "from tokenizers.pre_tokenizers import BertPreTokenizer\n",
    "from tokenizers.processors import BertProcessing\n",
    "from typing import Optional, List, Union\n",
    "\n",
    "from tokenizers.models import BPE\n",
    "from tokenizers.implementations.base_tokenizer import BaseTokenizer\n",
    "\n",
    "SPECIAL_CHARS = (\n",
    "    \n",
    "    # Emojis\n",
    "    \n",
    "    '©®‼⁉⃣™ℹ↔↕↖↗↘↙↩↪⌚⌛⌨⏏⏩⏪⏫⏬⏭⏮⏯⏰⏱⏲⏳⏸⏹⏺Ⓜ▪▫▶◀◻◼◽◾☀☁☂☃☄☎☑☔☕☘☝☠☢☣☦☪☮☯☸☹☺♀♂♈♉♊♋♌♍♎♏♐♑♒♓♟♠♣♥♦♨♻♾♿⚒⚓⚔⚕⚖⚗⚙⚛⚜⚠⚡⚧⚪⚫⚰⚱⚽⚾⛄⛅⛈⛎⛏⛑⛓⛔⛩⛪⛰⛱⛲⛳⛴⛵⛷⛸⛹⛺⛽✂✅✈✉✊✋✌✍✏✒✔✖✝✡✨✳✴❄❇❌❎❓❔❕❗❣❤➕➖➗➡➰➿⤴⤵⬅⬆⬇⬛⬜⭐⭕〰〽㊗㊙️🀄🃏🅰🅱🅾🅿🆎🆑🆒🆓🆔🆕🆖🆗🆘🆙🆚🇦🇧🇨🇩🇪🇫🇬🇭🇮🇯🇰🇱🇲🇳🇴🇵🇶🇷🇸🇹🇺🇻🇼🇽🇾🇿🈁🈂🈚🈯🈲🈳🈴🈵🈶🈷🈸🈹🈺🉐🉑🌀🌁🌂🌃🌄🌅🌆🌇🌈🌉🌊🌋🌌🌍🌎🌏🌐🌑🌒🌓🌔🌕🌖🌗🌘🌙🌚🌛🌜🌝🌞🌟🌠🌡🌤🌥🌦🌧🌨🌩🌪🌫🌬🌭🌮🌯🌰🌱🌲🌳🌴🌵🌶🌷🌸🌹🌺🌻🌼🌽🌾🌿🍀🍁🍂🍃🍄🍅🍆🍇🍈🍉🍊🍋🍌🍍🍎🍏🍐🍑🍒🍓🍔🍕🍖🍗🍘🍙🍚🍛🍜🍝🍞🍟🍠🍡🍢🍣🍤🍥🍦🍧🍨🍩🍪🍫🍬🍭🍮🍯🍰🍱🍲🍳🍴🍵🍶🍷🍸🍹🍺🍻🍼🍽🍾🍿🎀🎁🎂🎃🎄🎅🎆🎇🎈🎉🎊🎋🎌🎍🎎🎏🎐🎑🎒🎓🎖🎗🎙🎚🎛🎞🎟🎠🎡🎢🎣🎤🎥🎦🎧🎨🎩🎪🎫🎬🎭🎮🎯🎰🎱🎲🎳🎴🎵🎶🎷🎸🎹🎺🎻🎼🎽🎾🎿🏀🏁🏂🏃🏄🏅🏆🏇🏈🏉🏊🏋🏌🏍🏎🏏🏐🏑🏒🏓🏔🏕🏖🏗🏘🏙🏚🏛🏜🏝🏞🏟🏠🏡🏢🏣🏤🏥🏦🏧🏨🏩🏪🏫🏬🏭🏮🏯🏰🏳🏴🏵🏷🏸🏹🏺🐀🐁🐂🐃🐄🐅🐆🐇🐈🐉🐊🐋🐌🐍🐎🐏🐐🐑🐒🐓🐔🐕🐖🐗🐘🐙🐚🐛🐜🐝🐞🐟🐠🐡🐢🐣🐤🐥🐦🐧🐨🐩🐪🐫🐬🐭🐮🐯🐰🐱🐲🐳🐴🐵🐶🐷🐸🐹🐺🐻🐼🐽🐾🐿👀👁👂👃👄👅👆👇👈👉👊👋👌👍👎👏👐👑👒👓👔👕👖👗👘👙👚👛👜👝👞👟👠👡👢👣👤👥👦👧👨👩👪👫👬👭👮👯👰👱👲👳👴👵👶👷👸👹👺👻👼👽👾👿💀💁💂💃💄💅💆💇💈💉💊💋💌💍💎💏💐💑💒💓💔💕💖💗💘💙💚💛💜💝💞💟💠💡💢💣💤💥💦💧💨💩💪💫💬💭💮💯💰💱💲💳💴💵💶💷💸💹💺💻💼💽💾💿📀📁📂📃📄📅📆📇📈📉📊📋📌📍📎📏📐📑📒📓📔📕📖📗📘📙📚📛📜📝📞📟📠📡📢📣📤📥📦📧📨📩📪📫📬📭📮📯📰📱📲📳📴📵📶📷📸📹📺📻📼📽📿🔀🔁🔂🔃🔄🔅🔆🔇🔈🔉🔊🔋🔌🔍🔎🔏🔐🔑🔒🔓🔔🔕🔖🔗🔘🔙🔚🔛🔜🔝🔞🔟🔠🔡🔢🔣🔤🔥🔦🔧🔨🔩🔪🔫🔬🔭🔮🔯🔰🔱🔲🔳🔴🔵🔶🔷🔸🔹🔺🔻🔼🔽🕉🕊🕋🕌🕍🕎🕐🕑🕒🕓🕔🕕🕖🕗🕘🕙🕚🕛🕜🕝🕞🕟🕠🕡🕢🕣🕤🕥🕦🕧🕯🕰🕳🕴🕵🕶🕷🕸🕹🕺🖇🖊🖋🖌🖍🖐🖕🖖🖤🖥🖨🖱🖲🖼🗂🗃🗄🗑🗒🗓🗜🗝🗞🗡🗣🗨🗯🗳🗺🗻🗼🗽🗾🗿😀😁😂😃😄😅😆😇😈😉😊😋😌😍😎😏😐😑😒😓😔😕😖😗😘😙😚😛😜😝😞😟😠😡😢😣😤😥😦😧😨😩😪😫😬😭😮😯😰😱😲😳😴😵😶😷😸😹😺😻😼😽😾😿🙀🙁🙂🙃🙄🙅🙆🙇🙈🙉🙊🙋🙌🙍🙎🙏🚀🚁🚂🚃🚄🚅🚆🚇🚈🚉🚊🚋🚌🚍🚎🚏🚐🚑🚒🚓🚔🚕🚖🚗🚘🚙🚚🚛🚜🚝🚞🚟🚠🚡🚢🚣🚤🚥🚦🚧🚨🚩🚪🚫🚬🚭🚮🚯🚰🚱🚲🚳🚴🚵🚶🚷🚸🚹🚺🚻🚼🚽🚾🚿🛀🛁🛂🛃🛄🛅🛋🛌🛍🛎🛏🛐🛑🛒\\U0001f6d5\\U0001f6d6\\U0001f6d7🛠🛡🛢🛣🛤🛥🛩🛫🛬🛰🛳🛴🛵🛶\\U0001f6f7\\U0001f6f8\\U0001f6f9\\U0001f6fa\\U0001f6fb\\U0001f6fc\\U0001f7e0\\U0001f7e1\\U0001f7e2\\U0001f7e3\\U0001f7e4\\U0001f7e5\\U0001f7e6\\U0001f7e7\\U0001f7e8\\U0001f7e9\\U0001f7ea\\U0001f7eb\\U0001f90c\\U0001f90d\\U0001f90e\\U0001f90f🤐🤑🤒🤓🤔🤕🤖🤗🤘🤙🤚🤛🤜🤝🤞\\U0001f91f🤠🤡🤢🤣🤤🤥🤦🤧\\U0001f928\\U0001f929\\U0001f92a\\U0001f92b\\U0001f92c\\U0001f92d\\U0001f92e\\U0001f92f🤰\\U0001f931\\U0001f932🤳🤴🤵🤶🤷🤸🤹🤺🤼🤽🤾\\U0001f93f🥀🥁🥂🥃🥄🥅🥇🥈🥉🥊🥋\\U0001f94c\\U0001f94d\\U0001f94e\\U0001f94f🥐🥑🥒🥓🥔🥕🥖🥗🥘🥙🥚🥛🥜🥝🥞\\U0001f95f\\U0001f960\\U0001f961\\U0001f962\\U0001f963\\U0001f964\\U0001f965\\U0001f966\\U0001f967\\U0001f968\\U0001f969\\U0001f96a\\U0001f96b\\U0001f96c\\U0001f96d\\U0001f96e\\U0001f96f\\U0001f970\\U0001f971\\U0001f972\\U0001f973\\U0001f974\\U0001f975\\U0001f976\\U0001f977\\U0001f978\\U0001f97a\\U0001f97b\\U0001f97c\\U0001f97d\\U0001f97e\\U0001f97f🦀🦁🦂🦃🦄🦅🦆🦇🦈🦉🦊🦋🦌🦍🦎🦏🦐🦑\\U0001f992\\U0001f993\\U0001f994\\U0001f995\\U0001f996\\U0001f997\\U0001f998\\U0001f999\\U0001f99a\\U0001f99b\\U0001f99c\\U0001f99d\\U0001f99e\\U0001f99f\\U0001f9a0\\U0001f9a1\\U0001f9a2\\U0001f9a3\\U0001f9a4\\U0001f9a5\\U0001f9a6\\U0001f9a7\\U0001f9a8\\U0001f9a9\\U0001f9aa\\U0001f9ab\\U0001f9ac\\U0001f9ad\\U0001f9ae\\U0001f9af\\U0001f9b0\\U0001f9b1\\U0001f9b2\\U0001f9b3\\U0001f9b4\\U0001f9b5\\U0001f9b6\\U0001f9b7\\U0001f9b8\\U0001f9b9\\U0001f9ba\\U0001f9bb\\U0001f9bc\\U0001f9bd\\U0001f9be\\U0001f9bf🧀\\U0001f9c1\\U0001f9c2\\U0001f9c3\\U0001f9c4\\U0001f9c5\\U0001f9c6\\U0001f9c7\\U0001f9c8\\U0001f9c9\\U0001f9ca\\U0001f9cb\\U0001f9cd\\U0001f9ce\\U0001f9cf\\U0001f9d0\\U0001f9d1\\U0001f9d2\\U0001f9d3\\U0001f9d4\\U0001f9d5\\U0001f9d6\\U0001f9d7\\U0001f9d8\\U0001f9d9\\U0001f9da\\U0001f9db\\U0001f9dc\\U0001f9dd\\U0001f9de\\U0001f9df\\U0001f9e0\\U0001f9e1\\U0001f9e2\\U0001f9e3\\U0001f9e4\\U0001f9e5\\U0001f9e6\\U0001f9e7\\U0001f9e8\\U0001f9e9\\U0001f9ea\\U0001f9eb\\U0001f9ec\\U0001f9ed\\U0001f9ee\\U0001f9ef\\U0001f9f0\\U0001f9f1\\U0001f9f2\\U0001f9f3\\U0001f9f4\\U0001f9f5\\U0001f9f6\\U0001f9f7\\U0001f9f8\\U0001f9f9\\U0001f9fa\\U0001f9fb\\U0001f9fc\\U0001f9fd\\U0001f9fe\\U0001f9ff\\U0001fa70\\U0001fa71\\U0001fa72\\U0001fa73\\U0001fa74\\U0001fa78\\U0001fa79\\U0001fa7a\\U0001fa80\\U0001fa81\\U0001fa82\\U0001fa83\\U0001fa84\\U0001fa85\\U0001fa86\\U0001fa90\\U0001fa91\\U0001fa92\\U0001fa93\\U0001fa94\\U0001fa95\\U0001fa96\\U0001fa97\\U0001fa98\\U0001fa99\\U0001fa9a\\U0001fa9b\\U0001fa9c\\U0001fa9d\\U0001fa9e\\U0001fa9f\\U0001faa0\\U0001faa1\\U0001faa2\\U0001faa3\\U0001faa4\\U0001faa5\\U0001faa6\\U0001faa7\\U0001faa8\\U0001fab0\\U0001fab1\\U0001fab2\\U0001fab3\\U0001fab4\\U0001fab5\\U0001fab6\\U0001fac0\\U0001fac1\\U0001fac2\\U0001fad0\\U0001fad1\\U0001fad2\\U0001fad3\\U0001fad4\\U0001fad5\\U0001fad6\\U000e0062\\U000e0063\\U000e0065\\U000e0067\\U000e006c\\U000e006e\\U000e0073\\U000e0074\\U000e0077' \n",
    "    \n",
    "    # Simple Symbols\n",
    "    \n",
    "    #'><'\n",
    "    ',./?!@#_-=+~`\"\\';:$%*&^()[]{}'\n",
    "    \n",
    "    '、。《》「」『』|\\\\¶§⌘' \n",
    "    \n",
    "    # Fractions\n",
    "    \n",
    "    '⅟½⅓⅕⅙⅛⅔⅖⅚⅜¾⅗⅝⅞⅘¼⅐⅑⅒↉%℅‰‱'\n",
    "    \n",
    "    # Technicals\n",
    "    \n",
    "    '⌀⌂⌃⌄⌅⌆⌇⌈⌉⌊⌋⌌⌍⌎⌏⌐⌑⌒⌓⌔⌕⌖⌗⌘⌙⌚⌛⌜⌝⌞⌟⌠⌡⌢⌣⌤⌥⌦⌧⌨⌫⌬⌭⌮⌯⌰⌱⌲⌳⌴⌵⌶⌷⌸⌹⌺⌻⌼⌽⌾⌿⍀⍁⍂⍃⍄⍅⍆⍇⍈⍉⍊⍋⍌⍍⍎⍏⍐⍑⍒⍓⍔⍕⍖⍗⍘⍙⍚⍛⍜⍝⍞⍟⍠⍡⍢⍣⍤⍥⍦⍧⍨⍩⍪⍫⍬⍭⍮⍯⍰⍱⍲⍳⍴⍵⍶⍷⍸⍹⍺﹘﹝﹞﹟﹡〶␛␡␚␟␘␠␤␋␌␍␎␏␐␑␒␓␔␕␖␗␙␜␝␞␀␁␂␃␄␅␆␇␈␉␊␢␣⎋'\n",
    "    \n",
    "    # Rectangles\n",
    "    \n",
    "    '❏❐❑❒▀▁▂▃▄▅▆▇▉▊▋█▌▐▍▎▏▕░▒▓▔▬▢▣▤▥▦▧▨▩▪▫▭▮▯☰☲☱☴☵☶☳☷▰▱◧◨◩◪◫∎■□⊞⊟⊠⊡❘❙❚〓◊◈◇◆⎔⎚☖☗'\n",
    "    \n",
    "    # Triangles\n",
    "    \n",
    "    '◄▲▼►◀◣◥◤◢▶◂▴▾▸◁△▽▷∆∇⊳⊲⊴⊵◅▻▵▿◃▹◭◮⫷⫸⋖⋗⋪⋫⋬⋭⊿◬≜⑅'\n",
    "    \n",
    "    # Lines \n",
    "    \n",
    "    '│┃╽╿╏║╎┇︱┊︳┋┆╵〡〢╹╻╷〣☰☱☲☳☴☵☶☷≡✕═━─╍┅┉┄┈╌╴╶╸╺╼╾﹉﹍﹊﹎︲⑆⑇⑈⑉⑊⑄⑀︴﹏﹌﹋╳╲╱︶︵〵〴〳〆`ᐟ‐⁃⎯〄'\n",
    "    \n",
    "    # Corners\n",
    "    \n",
    "    '﹄﹃﹂﹁┕┓└┐┖┒┗┑┍┙┏┛┎┚┌┘「」『』˩˥├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╒╕╓╖╔╗╘╛╙╜╚╝╞╡╟╢╠╣╥╨╧╤╦╩╪╫╬〒⊢⊣⊤⊥╭╮╯╰⊦⊧⊨⊩⊪⊫⊬⊭⊮⊯⊺〦〧〨˦˧˨⑁⑂⑃∟'\n",
    "    \n",
    "    # Circles\n",
    "    \n",
    "    '◉○◌◍◎●◐◑◒◓◔◕◖◗❂☢⊗⊙◘◙◚◛◜◝◞◟◠◡◯〇〶⚫⬤◦∅∘⊕⊖⊘⊚⊛⊜⊝❍⦿'\n",
    "    \n",
    "    # Comparisons\n",
    "    \n",
    "    '≤≥≦≧≨≩⊰⊱⋛⋚≂≃≄≅≆≇≈≉≊≋≌≍≎≏≐≑≒≓≔≕≖≗≘≙≚≛≜≝≞≟≠≡≢≣'\n",
    "    \n",
    "    # Numerals\n",
    "    '⒈⒉⒊⒋⒌⒍⒎⒏⒐⒑⒒⒓⒔⒕⒖⒗⒘⒙⒚⒛⓿❶❷❸❹❺❻❼❽❾❿➀➁➂➃➄➅➆➇➈➉⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳⓪①②③④⑤⑥⑦⑧⑨⑩⓵⓶⓷⓸⓹⓺⓻⓼⓽⓾⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽⑾⑿⒀⒁⒂⒃⒄⒅⒆⒇➊➋➌➍➎➏➐➑➒➓⓫⓬⓭⓮⓯⓰⓱⓲⓳⓴'\n",
    "    \n",
    "    'ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫⅬⅭⅮⅯⅰⅱⅲⅳⅴⅵⅶⅷⅸⅹⅺⅻⅼⅽⅾⅿↀↁↂ➀➁➂➃➄➅➆➇➈➉➊➋➌➍➎➏➐➑➒➓⓵⓶⓷⓸⓹⓺⓻⓼⓽⓾⓿❶❷❸❹❺❻❼❽❾❿⁰¹²³⁴⁵⁶⁷⁸⁹₀₁₂₃₄₅₆₇₈₉⓪①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽⑾⑿⒀⒁⒂⒃⒄⒅⒆⒇⒈⒉⒊⒋⒌⒍⒎⒏⒐⒑⒒⒓⒔⒕⒖⒗⒘⒙⒚⒛㈠㈡㈢㈣㈤㈥㈦㈧㈨㈩㊀㊁㊂㊃㊄㊅㊆㊇㊈㊉０１２３４５６７８９ⁱₐₑₒₓₔ'\n",
    "    \n",
    "    # Currencies\n",
    "    \n",
    "    '₳฿￠₡¢₢₵₫€￡£₤₣ƒ₲₭₥₦₱＄$₮₩￦¥￥₴¤₰៛₪₯₠₧₨௹﷼৲৳₹' \n",
    "    \n",
    "    # Mathematic Symbols\n",
    "    \n",
    "    '∞⟀⟁⟂⟃⟄⟇⟈⟉⟊⟐⟑⟒⟓⟔⟕⟖⟗⟘⟙⟚⟛⟜⟝⟞⟟⟠⟡⟢⟣⟤⟥⟦⟧⟨⟩⟪⟫⦀⦁⦂⦃⦄⦅⦆⦇⦈⦉⦊⦋⦌⦍⦎⦏⦐⦑⦒⦓⦔⦕⦖⦗⦘⦙⦚⦛⦜⦝⦞⦟⦠⦡⦢⦣⦤⦥⦦⦧⦨⦩⦪⦫⦬⦭⦮⦯⦰⦱⦲⦳⦴⦵⦶⦷⦸⦹⦺⦻⦼⦽⦾⦿⧀⧁⧂⧃⧄⧅⧆⧇⧈⧉⧊⧋⧌⧍⧎⧏⧐⧑⧒⧓⧔⧕⧖⧗⧘⧙⧚⧛⧜⧝⧞⧟⧡⧢⧣⧤⧥⧦⧧⧨⧩⧪⧫⧬⧭⧮⧯⧰⧱⧲⧳⧴⧵⧶⧷⧸⧹⧺⧻⧼⧽⧾⧿∀∁∂∃∄∅∆∇∈∉∊∋∌∍∎∏∐∑−∓∔∕∖∗∘∙√∛∜∝∟∠∡∢∣∤∥∦∧∨∩∪∫∬∭∮∯∰∱∲∳∴∵∶∷∸∹∺∻∼∽∾∿≀≁≂≃≄≅≆≇≈≉≊≋≌≍≎≏≐≑≒≓≔≕≖≗≘≙≚≛≜≝≞≟≠≡≢≣≤≥≦≧≨≩≪≫≬≭≰≱≲≳≴≵≶≷≸≹≺≻≼≽≾≿⊀⊁⊂⊃⊄⊅⊆⊇⊈⊉⊊⊋⊌⊍⊎⊏⊐⊑⊒⊓⊔⊕⊖⊗⊘⊙⊚⊛⊜⊝⊞⊟⊠⊡⊢⊣⊤⊥⊦⊧⊨⊩⊪⊫⊬⊭⊮⊯⊰⊱⊲⊳⊴⊵⊶⊷⊸⊹⊺⊻⊼⊽⊾⊿⋀⋁⋂⋃⋄⋅⋆⋇⋈⋉⋊⋋⋌⋍⋎⋏⋐⋑⋒⋓⋔⋕⋖⋗⋘⋙⋚⋛⋜⋝⋞⋟⋠⋡⋢⋣⋤⋥⋦⋧⋨⋩⋪⋫⋬⋭⋮⋯⋰⋱⋲⋳⋴⋵⋶⋷⋸⋹⋺⋻⋼⋽⋾⋿✕✖✚÷°'\n",
    "    \n",
    "    # Maths\n",
    "    \n",
    "    'π∞Σ√∛∜∫∬∭∮∯∰∱∲∳∀∁∂∃∄∅∆∇∈∉∊∋∌∍∎∏∐∑−∓∔∕∖∗∘∙∝∟∠∡∢∣∤∥∦∧∨∩∪∴∵∶∷∸∹∺∻∼∽∾∿≀≁≂≃≄≅≆≇≈≉≊≋≌≍≎≏≐≑≒≓≔≕≖≗≘≙≚≛≜≝≞≟≠≡≢≣≤≥≦≧≨≩≪≫≬≭≰≱≲≳≴≵≶≷≸≹≺≻≼≽≾≿⊀⊁⊂⊃⊄⊅⊆⊇⊈⊉⊊⊋⊌⊍⊎⊏⊐⊑⊒⊓⊔⊕⊖⊗⊘⊙⊚⊛⊜⊝⊞⊟⊠⊡⊢⊣⊤⊥⊦⊧⊨⊩⊪⊫⊬⊭⊮⊯⊰⊱⊲⊳⊴⊵⊶⊷⊸⊹⊺⊻⊼⊽⊾⊿⋀⋁⋂⋃⋄⋅⋆⋇⋈⋉⋊⋋⋌⋍⋎⋏⋐⋑⋒⋓⋔⋕⋖⋗⋘⋙⋚⋛⋜⋝⋞⋟⋠⋡⋢⋣⋤⋥⋦⋧⋨⋩⋪⋫⋬⋭⋮⋯⋰⋱⁺⁻⁼⁽⁾ⁿ₊₋₌₍₎✖﹢﹣＋－／＝÷±×',\n",
    "    \n",
    "    # Braille Patterns\n",
    "    '⠁⠂⠄⠈⠐⠠⠃⠅⠆⠘⠨⠰⠉⠒⠤⠑⠡⠢⠊⠌⠔⠇⠸⠎⠱⠣⠜⠪⠕⠋⠙⠓⠚⠍⠩⠥⠬⠖⠲⠦⠴⠏⠹⠧⠼⠫⠝⠮⠵⠺⠗⠞⠳⠛⠭⠶⠟⠻⠷⠾⠯⠽⠿⡀⡄⡆⡇⡏⡛⡜⡟⡶⡷⡼⡾⡿⢀⢉⢠⢣⢤⢧⢰⢸⢹⢻⢿⣀⣁⣄⣆⣇⣉⣒⣕⣘⣙⣛⣠⣤⣥⣦⣧⣩⣬⣭⣰⣴⣵⣶⣷⣸⣹⣻⣼⣾⣿'\n",
    "    \n",
    "    # Zhuyin\n",
    "    \n",
    "    'ㄍㄎㄫㄐㄑㄬㄉㄊㄋㄅㄆㄇㄈㄪㄗㄘㄙㄓㄔㄕㄏㄒㄌㄖㄧㄨㄩㄚㄛㄝㄟㄞㄠㄡㄢㄤㄣㄥㄦ'\n",
    "    \n",
    "    # Gender\n",
    "    \n",
    "    '♀♂☹☺☻☿〠ヅツ㋡웃유üÜتシッ㋛웃̟͟ꑇꐦꐠꐡꐕꌇꌈꉕꈋꈌꆛꆜꃼ☠☃〲〴ϡﭢ⍢⍣⍤⍥⍨⍩ὃὕὣѶӪӫ⚣⚤⚥⚦⚧⚨⚢'\n",
    "    \n",
    "    # Musical Symbols\n",
    "    \n",
    "    '♩♪♫♬♭♮♯°ø؂≠≭'\n",
    "    \n",
    "    # Punctuations\n",
    "    \n",
    "    '·‑‒–—―‗‘’‚‛“”„‟•‣․‥…‧′″‴‵‶‷❛❜❝❞ʹʺʻʼʽʾʿˀˁ˂˃˄˅ˆˇˈˉˊˋˌˍˎˏːˑ˒˓˔˕˖˗˘˙˚˛˜˝˞ˠˡ～¿﹐﹒﹔﹕！＃＄％＆＊，．：；？＠、。〃〝〞︰'\n",
    "    \n",
    "    # Ticks / Cross\n",
    "    \n",
    "    '✓✔✗✘☓∨√✇☐☑☒〤〥',\n",
    "\n",
    "    # Stars\n",
    "\n",
    "    '★☆≛⋆⍟⍣★☆✡✦✧✪✫✬✯✰✴✵✶✷✸',\n",
    "    \n",
    "    # Hearts\n",
    "    \n",
    "    '♥♡❤❥❣❦❧დღ۵ლওლ❤️️💙🧡💚💛💜🖤💗💓💔💟💕💖❣️💘💝💞'\n",
    "    \n",
    "    # Astrological & Zodiac Sign Symbols\n",
    "    \n",
    "    '☮☸♈♉☪♊♋♌♍♎♏♐♑♒♓☤☥☧☨☩☫☬☭☯☽☾✙✚✛✜✝✞✟†⊹‡♁♆❖♅✠✡✢卍卐〷☠☢☣☦'\n",
    "\n",
    "    # Flowers\n",
    "\n",
    "    '✽✾✿❀❁❃❊❋✤✣⚘⚜ꕤꕥ☘'\n",
    "    \n",
    "    # Arrows\n",
    "         '☚👈☛👉🖝🖜🖛🖚☜☞🖢👆🖞☝🖣👇🖟☟↕↖↗↘↙↚↛↜↝↞↟↠↡↢↣↤↥↦↧↨↩↪↫↬↭↮↯↰↱↲↳↴↶↷↸↹↺↻↼↽↾↿⇀⇁⇂⇃⇄⇅⇆⇇⇈⇉⇊⇋⇌⇍⇎⇏⇕⇖⇗⇘⇙⇚⇛⇜⇝⇞⇟⇠⇡⇢⇣⇤⇥⇦⇧⇨⇩⇪⌅⌆⌤⏎▶☇☈☊☋☌☍➔➘➙➚➛➜➝➞➟➠➡➢➣➤➥➦➧➨➩➪➫➬➭➮➯➱➲➳➴➵➶➷➸➹➺➻➼➽➾⤴⤵↵↓↔←→↑⌦⌫⌧⇰⇫⇬⇭⇳⇮⇯⇱⇲⇴⇵⇷⇸⇹⇺⇑⇓⇽⇾⇿⬳⟿⤉⤈⇻⇼⬴⤀⬵⤁⬹⤔⬺⤕⬶⤅⬻⤖⬷⤐⬼⤗⬽⤘⤝⤞⤟⤠⤡⤢⤣⤤⤥⤦⤪⤨⤧⤩⤭⤮⤯⤰⤱⤲⤫⤬⬐⬎⬑⬏⤶⤷⥂⥃⥄⭀⥱⥶⥸⭂⭈⭊⥵⭁⭇⭉⥲⭋⭌⥳⥴⥆⥅⥹⥻⬰⥈⬾⥇⬲⟴⥷⭃⥺⭄⥉⥰⬿⤳⥊⥋⥌⥍⥎⥏⥐⥑⥒⥓⥔⥕⥖⥗⥘⥙⥚⥛⥜⥝⥞⥟⥠⥡⥢⥤⥣⥥⥦⥨⥧⥩⥮⥯⥪⥬⥫⥭⤌⤍⤎⤏⬸⤑⬱⟸⟹⟺⤂⤃⤄⤆⤇⤊⤋⭅⭆⟰⟱⇐⇒⇔⇶⟵⟶⟷⬄⬀⬁⬂⬃⬅⬆⬇⬈⬉⬊⬋⬌⬍⟻⟼⤒⤓⤙⤚⤛⤜⥼⥽⥾⥿⤼⤽⤾⤿⤸⤺⤹⤻⥀⥁⟲⟳'\n",
    "    \n",
    "    # Weather related symbols\n",
    "    \n",
    "    '°℃℉ϟ☀☁☂☃☉☼☽☾♁♨❄❅❆☇☈☄㎎㎏㎜㎝㎞㎡㏄㏎㏑㏒㏕'\n",
    "    \n",
    "    # Others\n",
    "    \n",
    "    '⋮⋱ↀↁↂ✿✽✻✰✩✦✧♕♕ᐛ♚ᴥᴗᗜㄜ꒪'\n",
    "\n",
    "    # Emoji Skin Tone\n",
    "\n",
    "    '🏻🏼🏽🏾🏿'\n",
    "\n",
    "    # Japanese Characters\n",
    "\n",
    "    'あいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをんゝアイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワヲンヶーヽ'\n",
    "\n",
    "    # Pinyin\n",
    "\n",
    "    'ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩㄱ'\n",
    "\n",
    "    '\\u200d'\n",
    "\n",
    ")\n",
    "\n",
    "SPECIAL_CHARS = ''.join(sorted(set(SPECIAL_CHARS)))\n",
    "\n",
    "class CanTokenizer(BaseTokenizer):\n",
    "    \"\"\" Uses Bert WordPiece Tokenizer \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        vocab_file: Optional[str] = None,\n",
    "        unk_token: Union[str, AddedToken] = \"<unk>\",\n",
    "        sep_token: Union[str, AddedToken] = \"</s>\",\n",
    "        cls_token: Union[str, AddedToken] = \"<s>\",\n",
    "        nl_token: Union[str, AddedToken] = \"<nl>\",\n",
    "        pad_token: Union[str, AddedToken] = \"<pad>\",\n",
    "        mask_token: Union[str, AddedToken] = \"<mask>\",\n",
    "        clean_text: bool = True,\n",
    "        handle_chinese_chars: bool = True,\n",
    "        separate_numbers: bool = True,\n",
    "        strip_accents: bool = True,\n",
    "        lowercase: bool = True,\n",
    "        wordpieces_prefix: str = \"##\",\n",
    "        special_chars: str = SPECIAL_CHARS,\n",
    "        zh_norm: bool = True,\n",
    "        handle_simpl: bool = True,\n",
    "        do_postprocess: bool = False\n",
    "    ):\n",
    "\n",
    "        if vocab_file is not None:\n",
    "            tokenizer = Tokenizer(WordPiece(vocab_file, unk_token=str(unk_token)))\n",
    "        else:\n",
    "            tokenizer = Tokenizer(WordPiece())\n",
    "\n",
    "        # Let the tokenizer know about special tokens if they are part of the vocab\n",
    "        if tokenizer.token_to_id(str(unk_token)) is not None:\n",
    "            tokenizer.add_special_tokens([str(unk_token)])\n",
    "        if tokenizer.token_to_id(str(sep_token)) is not None:\n",
    "            tokenizer.add_special_tokens([str(sep_token)])\n",
    "        if tokenizer.token_to_id(str(cls_token)) is not None:\n",
    "            tokenizer.add_special_tokens([str(cls_token)])\n",
    "        if tokenizer.token_to_id(str(pad_token)) is not None:\n",
    "            tokenizer.add_special_tokens([str(pad_token)])\n",
    "        if tokenizer.token_to_id(str(nl_token)) is not None:\n",
    "            tokenizer.add_special_tokens([str(nl_token)])\n",
    "        if tokenizer.token_to_id(str(mask_token)) is not None:\n",
    "            tokenizer.add_special_tokens([str(mask_token)])\n",
    "        if tokenizer.token_to_id(str(mask_token)) is not None:\n",
    "            tokenizer.add_special_tokens([str(mask_token)])\n",
    "\n",
    "        tokenizer.normalizer = Sequence([NFKC(), BertNormalizer(\n",
    "            clean_text=clean_text,\n",
    "            handle_chinese_chars=handle_chinese_chars,\n",
    "            separate_numbers=separate_numbers,\n",
    "            strip_accents=strip_accents,\n",
    "            lowercase=lowercase,\n",
    "            special_chars=special_chars,\n",
    "            zh_norm=zh_norm,\n",
    "            handle_simpl=handle_simpl\n",
    "        )])\n",
    "        tokenizer.pre_tokenizer = BertPreTokenizer()\n",
    "\n",
    "        if vocab_file is not None and do_postprocess:\n",
    "            sep_token_id = tokenizer.token_to_id(str(sep_token))\n",
    "            if sep_token_id is None:\n",
    "                raise TypeError(\"sep_token not found in the vocabulary\")\n",
    "            cls_token_id = tokenizer.token_to_id(str(cls_token))\n",
    "            if cls_token_id is None:\n",
    "                raise TypeError(\"cls_token not found in the vocabulary\")\n",
    "            tokenizer.post_processor = BertProcessing(\n",
    "                (str(sep_token), sep_token_id), (str(cls_token), cls_token_id)\n",
    "            )\n",
    "        \n",
    "        tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)\n",
    "\n",
    "        parameters = {\n",
    "            \"model\": \"BertWordPiece\",\n",
    "            \"unk_token\": unk_token,\n",
    "            \"sep_token\": sep_token,\n",
    "            \"cls_token\": cls_token,\n",
    "            \"nl_token\": nl_token,\n",
    "            \"pad_token\": pad_token,\n",
    "            \"mask_token\": mask_token,\n",
    "            \"clean_text\": clean_text,\n",
    "            \"handle_chinese_chars\": handle_chinese_chars,\n",
    "            \"separate_numbers\": separate_numbers,\n",
    "            \"strip_accents\": strip_accents,\n",
    "            \"lowercase\": lowercase,\n",
    "            \"special_chars\": special_chars,\n",
    "            \"zh_norm\": zh_norm,\n",
    "            \"handle_simpl\": handle_simpl,\n",
    "            \"wordpieces_prefix\": wordpieces_prefix,\n",
    "        }\n",
    "\n",
    "        super().__init__(tokenizer, parameters)\n",
    "\n",
    "    def train(\n",
    "        self,\n",
    "        files: Union[str, List[str]],\n",
    "        vocab_size: int = 30000,\n",
    "        min_frequency: int = 20,\n",
    "        limit_alphabet: int = 6000,\n",
    "        initial_alphabet: List[str] = [],\n",
    "        special_tokens: List[Union[str, AddedToken]] = [\n",
    "            \"<pad>\",\n",
    "            \"<unk>\",\n",
    "            \"<s>\",\n",
    "            \"<nl>\",\n",
    "            \"</s>\",\n",
    "            \"<mask>\",\n",
    "        ],\n",
    "        show_progress: bool = True,\n",
    "        wordpieces_prefix: str = \"##\",\n",
    "    ):\n",
    "        \"\"\" Train the model using the given files \"\"\"\n",
    "\n",
    "        trainer = trainers.WordPieceTrainer(\n",
    "            vocab_size=vocab_size,\n",
    "            min_frequency=min_frequency,\n",
    "            limit_alphabet=limit_alphabet,\n",
    "            initial_alphabet=initial_alphabet,\n",
    "            special_tokens=special_tokens,\n",
    "            show_progress=show_progress,\n",
    "            continuing_subword_prefix=wordpieces_prefix,\n",
    "        )\n",
    "        if isinstance(files, str):\n",
    "            files = [files]\n",
    "        self._tokenizer.train(trainer, files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}