Transformer Implementation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "Why Transformers?\n",
    "- superior performance modeling sequential data\n",
    "- parrallelizable -> much faster on GPUs\n",
    "- capture long-distance dependencies\n",
    "- learned temporal or spatial relationships"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-06-26T09:47:17.200908Z",
     "start_time": "2023-06-26T09:46:44.739373600Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "tensorboard 2.10.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 4.23.3 which is incompatible.\n",
      "tensorflow-gpu 2.10.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 4.23.3 which is incompatible.\n",
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "tensorflow-datasets 4.9.2 requires protobuf>=3.20, but you have protobuf 3.19.6 which is incompatible.\n",
      "tensorflow-metadata 1.13.1 requires protobuf<5,>=3.20.3, but you have protobuf 3.19.6 which is incompatible.\n"
     ]
    }
   ],
   "source": [
    "! pip install -q tensorflow_datasets\n",
    "! pip install -q -U tensorflow-text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'builder' from 'google.protobuf.internal' (C:\\Users\\WG\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\google\\protobuf\\internal\\__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mImportError\u001B[0m                               Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[3], line 5\u001B[0m\n\u001B[0;32m      3\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mnumpy\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01mnp\u001B[39;00m\n\u001B[0;32m      4\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mmatplotlib\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mpyplot\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01mplt\u001B[39;00m\n\u001B[1;32m----> 5\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01mtfds\u001B[39;00m\n\u001B[0;32m      6\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mtensorflow\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01mtf\u001B[39;00m\n\u001B[0;32m      7\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mtensorflow_text\u001B[39;00m\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\__init__.py:43\u001B[0m\n\u001B[0;32m     41\u001B[0m _TIMESTAMP_IMPORT_STARTS \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[0;32m     42\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mabsl\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m logging\n\u001B[1;32m---> 43\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mlogging\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01m_tfds_logging\u001B[39;00m\n\u001B[0;32m     44\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mlogging\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m call_metadata \u001B[38;5;28;01mas\u001B[39;00m _call_metadata\n\u001B[0;32m     46\u001B[0m _metadata \u001B[38;5;241m=\u001B[39m _call_metadata\u001B[38;5;241m.\u001B[39mCallMetadata()\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\__init__.py:22\u001B[0m\n\u001B[0;32m     18\u001B[0m \u001B[38;5;66;03m# Allow to use `tfds.core.Path` in dataset implementation which seems more\u001B[39;00m\n\u001B[0;32m     19\u001B[0m \u001B[38;5;66;03m# natural than having to import a third party module.\u001B[39;00m\n\u001B[0;32m     20\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01metils\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mepath\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Path\n\u001B[1;32m---> 22\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m community\n\u001B[0;32m     23\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdataset_builder\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m BeamBasedBuilder\n\u001B[0;32m     24\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdataset_builder\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m BuilderConfig\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\community\\__init__.py:18\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# coding=utf-8\u001B[39;00m\n\u001B[0;32m      2\u001B[0m \u001B[38;5;66;03m# Copyright 2023 The TensorFlow Datasets Authors.\u001B[39;00m\n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m#\u001B[39;00m\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m     13\u001B[0m \u001B[38;5;66;03m# See the License for the specific language governing permissions and\u001B[39;00m\n\u001B[0;32m     14\u001B[0m \u001B[38;5;66;03m# limitations under the License.\u001B[39;00m\n\u001B[0;32m     16\u001B[0m \u001B[38;5;124;03m\"\"\"Community dataset API.\"\"\"\u001B[39;00m\n\u001B[1;32m---> 18\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcommunity\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mhuggingface_wrapper\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m mock_builtin_to_use_gfile\n\u001B[0;32m     19\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcommunity\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mhuggingface_wrapper\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m mock_huggingface_import\n\u001B[0;32m     20\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcommunity\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mload\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m builder_cls_from_module\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\community\\huggingface_wrapper.py:31\u001B[0m\n\u001B[0;32m     28\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01munittest\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m mock\n\u001B[0;32m     30\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01metils\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m epath\n\u001B[1;32m---> 31\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dataset_builder\n\u001B[0;32m     32\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dataset_info\n\u001B[0;32m     33\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m download\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\dataset_builder.py:34\u001B[0m\n\u001B[0;32m     32\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01metils\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m epath\n\u001B[0;32m     33\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m constants\n\u001B[1;32m---> 34\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dataset_info\n\u001B[0;32m     35\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dataset_metadata\n\u001B[0;32m     36\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m decode\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\dataset_info.py:50\u001B[0m\n\u001B[0;32m     48\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m lazy_imports_lib\n\u001B[0;32m     49\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m naming\n\u001B[1;32m---> 50\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m splits \u001B[38;5;28;01mas\u001B[39;00m splits_lib\n\u001B[0;32m     51\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m utils\n\u001B[0;32m     52\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mfeatures\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m feature \u001B[38;5;28;01mas\u001B[39;00m feature_lib\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\splits.py:34\u001B[0m\n\u001B[0;32m     32\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01metils\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m epath\n\u001B[0;32m     33\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m naming\n\u001B[1;32m---> 34\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m proto \u001B[38;5;28;01mas\u001B[39;00m proto_lib\n\u001B[0;32m     35\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m units\n\u001B[0;32m     36\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m utils\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\proto\\__init__.py:18\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# coding=utf-8\u001B[39;00m\n\u001B[0;32m      2\u001B[0m \u001B[38;5;66;03m# Copyright 2023 The TensorFlow Datasets Authors.\u001B[39;00m\n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m#\u001B[39;00m\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m     13\u001B[0m \u001B[38;5;66;03m# See the License for the specific language governing permissions and\u001B[39;00m\n\u001B[0;32m     14\u001B[0m \u001B[38;5;66;03m# limitations under the License.\u001B[39;00m\n\u001B[0;32m     16\u001B[0m \u001B[38;5;124;03m\"\"\"Public API of the proto package.\"\"\"\u001B[39;00m\n\u001B[1;32m---> 18\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mproto\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dataset_info_generated_pb2 \u001B[38;5;28;01mas\u001B[39;00m dataset_info_pb2  \u001B[38;5;66;03m# pylint: disable=line-too-long\u001B[39;00m\n\u001B[0;32m     19\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtensorflow_datasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mproto\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m feature_generated_pb2 \u001B[38;5;28;01mas\u001B[39;00m feature_pb2  \u001B[38;5;66;03m# pylint: disable=line-too-long\u001B[39;00m\n\u001B[0;32m     21\u001B[0m SplitInfo \u001B[38;5;241m=\u001B[39m dataset_info_pb2\u001B[38;5;241m.\u001B[39mSplitInfo\n",
      "File \u001B[1;32m~\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\tensorflow_datasets\\core\\proto\\dataset_info_generated_pb2.py:22\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# coding=utf-8\u001B[39;00m\n\u001B[0;32m      2\u001B[0m \u001B[38;5;66;03m# Copyright 2023 The TensorFlow Datasets Authors.\u001B[39;00m\n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m#\u001B[39;00m\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m     19\u001B[0m \u001B[38;5;66;03m# Generated by the protocol buffer compiler.  DO NOT EDIT!\u001B[39;00m\n\u001B[0;32m     20\u001B[0m \u001B[38;5;66;03m# source: dataset_info.proto\u001B[39;00m\n\u001B[0;32m     21\u001B[0m \u001B[38;5;124;03m\"\"\"Generated protocol buffer code.\"\"\"\u001B[39;00m\n\u001B[1;32m---> 22\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mgoogle\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mprotobuf\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01minternal\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m builder \u001B[38;5;28;01mas\u001B[39;00m _builder\n\u001B[0;32m     23\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mgoogle\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mprotobuf\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m descriptor \u001B[38;5;28;01mas\u001B[39;00m _descriptor\n\u001B[0;32m     24\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mgoogle\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mprotobuf\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m descriptor_pool \u001B[38;5;28;01mas\u001B[39;00m _descriptor_pool\n",
      "\u001B[1;31mImportError\u001B[0m: cannot import name 'builder' from 'google.protobuf.internal' (C:\\Users\\WG\\miniconda3\\envs\\PML_Projekt\\lib\\site-packages\\google\\protobuf\\internal\\__init__.py)"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "import time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow_datasets as tfds\n",
    "import tensorflow as tf\n",
    "import tensorflow_text"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-26T09:47:20.129418700Z",
     "start_time": "2023-06-26T09:47:19.388991500Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Dataset\n",
    "This is not needed for our later Fingerspelling Recognition system but only for testing the model during development."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tfds' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[4], line 1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m examples, metadata \u001B[38;5;241m=\u001B[39m \u001B[43mtfds\u001B[49m\u001B[38;5;241m.\u001B[39mload(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mted_hrlr_translate/pt_to_en\u001B[39m\u001B[38;5;124m'\u001B[39m,\n\u001B[0;32m      2\u001B[0m                                with_info\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m,\n\u001B[0;32m      3\u001B[0m                                as_supervised\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[0;32m      4\u001B[0m train_examples, val_examples \u001B[38;5;241m=\u001B[39m examples[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mtrain\u001B[39m\u001B[38;5;124m'\u001B[39m], examples[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mvalidation\u001B[39m\u001B[38;5;124m'\u001B[39m]\n",
      "\u001B[1;31mNameError\u001B[0m: name 'tfds' is not defined"
     ]
    }
   ],
   "source": [
    "examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en',\n",
    "                               with_info=True,\n",
    "                               as_supervised=True)\n",
    "train_examples, val_examples = examples['train'], examples['validation']"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-26T09:47:26.988065900Z",
     "start_time": "2023-06-26T09:47:26.932509900Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "tokenize words"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model_name='ted_hrlr_translate_pt_en_converter'\n",
    "tf.keras.utils.get_file(\n",
    "    f'{model_name}.zip',\n",
    "    f'https://storage.googleapis.com/download.tensorflow.org/models/{model_name}.zip',\n",
    "    cache_dir='.', cache_subdir='', extract=True\n",
    ")\n",
    "tokenizers = tf.saved_model.load(model_name)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "MAX_TOKENS = 128\n",
    "def prepare_batch(pt, en):\n",
    "    pt = tokenizers.pt.tokenize(pt)\n",
    "    pt = pt[:, :MAX_TOKENS]\n",
    "    pt = pt.to_tensor()\n",
    "\n",
    "    en = tokenizers.en.tokenize(en)\n",
    "    en = en[:, :(MAX_TOKENS+1)]\n",
    "    en_inputs = en[:, :-1].to_tensor()\n",
    "    en_labels = en[:, 1:].to_tensor()\n",
    "    return (pt, en_inputs), en_labels"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "BUFFER_SIZE = 2000\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "def make_batches(ds):\n",
    "    return (\n",
    "        ds\n",
    "            .shuffle(BUFFER_SIZE)\n",
    "            .batch(BATCH_SIZE)\n",
    "            .map(prepare_batch, tf.data.AUTOTUNE)\n",
    "            .prefetch(buffer_size=tf.data.AUTOTUNE)\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "train_batches = make_batches(train_examples)\n",
    "val_batches = make_batches(val_examples)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Define Transformer Components\n",
    "\n",
    "## Embedding and Positional Encoding"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def positional_encoding(length, depth):\n",
    "    depth = depth/2\n",
    "\n",
    "    positions = np.arange(length)[:, np.newaxis]\n",
    "    depths = np.arange(depth)[np.newaxis, :]/depth\n",
    "\n",
    "    angle_rates = 1/(10000**depths)\n",
    "    angle_rads = positions * angle_rates\n",
    "\n",
    "    pos_encoding = np.concatenate(\n",
    "        [np.sin(angle_rads), np.cos(angle_rads)],\n",
    "        axis=-1\n",
    "    )\n",
    "\n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "I am unsure as of now as we need to apply an Embedding layer to the mediapipe data.\n",
    "I am also unsure if we need it for the sequence of letters.\n",
    "But we definitly need positional encoding for both."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class PositionalEmbedding(tf.keras.layers.Layer):\n",
    "    def __init__(self, vocab_size, d_model, is_mp: bool):\n",
    "        super().__init__()\n",
    "        self.d_model = d_model\n",
    "        if not is_mp:\n",
    "            self.embedding = tf.keras.layers.Embedding(vocab_size, d_model, mask_zero=True)\n",
    "        self.pos_encoding = positional_encoding(length=2048, depth=d_model)\n",
    "        self.is_mp = is_mp\n",
    "\n",
    "    def compute_mask(self, *args, **kwargs):\n",
    "        return self.embedding.compute_mask(*args, **kwargs)\n",
    "\n",
    "    def call(self, x):\n",
    "        length = tf.shape(x)[1]\n",
    "        if not self.is_mp:\n",
    "            x = self.embedding(x)\n",
    "            x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x = x + self.pos_encoding[tf.newaxis, :length, :]\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Attention layers\n",
    "\n",
    "### Base Attention class"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class BaseAttention(tf.keras.layers.Layer):\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__()\n",
    "        self.mha = tf.keras.layers.MultiHeadAttention(**kwargs)\n",
    "        self.layernorm = tf.keras.layers.LayerNormalization()\n",
    "        self.add = tf.keras.layers.Add()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Cross Attention"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class CrossAttention(BaseAttention):\n",
    "\n",
    "    def call(self, x, context):\n",
    "        attn_output, attn_scores = self.mha(\n",
    "            query=x,\n",
    "            key=context,\n",
    "            value=context,\n",
    "            return_attention_scores=True\n",
    "        )\n",
    "        self.last_attn_scores = attn_scores\n",
    "\n",
    "        x = self.add([x, attn_output])\n",
    "        x = self.layernorm(x)\n",
    "\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Global Self Attention"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class GlobalSelfAttention(BaseAttention):\n",
    "\n",
    "    def call(self, x):\n",
    "        attn_output = self.mha(\n",
    "            query=x,\n",
    "            value=x,\n",
    "            key=x\n",
    "        )\n",
    "        x = self.add([x, attn_output])\n",
    "        x = self.layernorm(x)\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Causal (Masked) Self Attention"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class CausalSelfAttention(BaseAttention):\n",
    "\n",
    "    def call(self, x):\n",
    "        attn_output = self.mha(\n",
    "            query=x,\n",
    "            value=x,\n",
    "            key=x,\n",
    "            use_causal_mask=True\n",
    "        )\n",
    "        x = self.add([x, attn_output])\n",
    "        x = self.layernorm(x)\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Feed-Forward Network"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class FeedForward(tf.keras.layers.Layer):\n",
    "\n",
    "    def __init__(self, d_model, dff, dropout_rate=0.1):\n",
    "        super().__init__()\n",
    "        self.seq = tf.keras.Sequential([\n",
    "            tf.keras.layers.Dense(dff, activation='relu'),\n",
    "            tf.keras.layers.Dense(d_model),\n",
    "            tf.keras.layers.Dropout(dropout_rate)\n",
    "        ])\n",
    "        self.add = tf.keras.layers.Add()\n",
    "        self.layer_norm = tf.keras.layers.LayerNormalization()\n",
    "\n",
    "    def call(self, x):\n",
    "        x = self.add([x, self.seq(x)])\n",
    "        x = self.layer_norm(x)\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Encoder Block"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class EncoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, *, d_model, num_heads, dff, dropout_rate=0.1):\n",
    "        super().__init__()\n",
    "\n",
    "        self.self_attention = GlobalSelfAttention(\n",
    "            num_heads=num_heads,\n",
    "            key_dim=d_model,\n",
    "            dropout=dropout_rate\n",
    "        )\n",
    "        self.ffn = FeedForward(d_model, dff)\n",
    "\n",
    "    def call(self, x, *args, **kwargs):\n",
    "        x = self.self_attention(x)\n",
    "        x = self.ffn(x)\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "\n",
    "    def __init__(self, *, num_layers, d_model, num_heads, dff, vocab_size, droput_rate=0.1):\n",
    "        super().__init__()\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.pos_embedding = PositionalEmbedding(\n",
    "            vocab_size=vocab_size, d_model=d_model, is_mp=True\n",
    "        )\n",
    "\n",
    "        self.enc_layers = [\n",
    "            EncoderLayer(d_model=d_model,\n",
    "                         num_heads=num_heads,\n",
    "                         dff=dff,\n",
    "                         dropout_rate=droput_rate)\n",
    "        for _ in range(num_layers)]\n",
    "\n",
    "        self.dropout = tf.keras.layers.Dropout(droput_rate)\n",
    "\n",
    "    def call(self, x, *args, **kwargs):\n",
    "        x = self.pos_embedding(x)\n",
    "\n",
    "        x = self.dropout(x)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            x = self.enc_layers[i](x)\n",
    "\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Decoder Block"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "\n",
    "    def __init__(self, *, d_model, num_heads, dff, dropout_rate=0.1):\n",
    "        super().__init__()\n",
    "\n",
    "        self.causal_self_attention = CausalSelfAttention(\n",
    "            num_heads=num_heads,\n",
    "            key_dim=d_model,\n",
    "            dropout_rate=dropout_rate\n",
    "        )\n",
    "\n",
    "        self.cross_attention = CrossAttention(\n",
    "            num_heads=num_heads,\n",
    "            key_dim=d_model,\n",
    "            dropout_rate=dropout_rate\n",
    "        )\n",
    "\n",
    "        self.ffn = FeedForward(d_model=d_model, dff=dff, dropout_rate=dropout_rate)\n",
    "\n",
    "    def call(self, x, context, *args, **kwargs):\n",
    "        x = self.causal_self_attention(x=x)\n",
    "        x = self.cross_attention(x=x, context=context)\n",
    "        self.last_attn_scores = self.cross_attention.last_attn_scores\n",
    "\n",
    "        x = self.ffn(x)\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "\n",
    "    def __init__(self, *, num_layers, d_model, num_heads, dff, vocab_size, dropout_rate=0.1):\n",
    "        super(Decoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "        self.pos_embedding = PositionalEmbedding(vocab_size=vocab_size, d_model=d_model, is_mp=False)\n",
    "        self.dropout = tf.keras.layers.Dropout(dropout_rate)\n",
    "        self.dec_layers = [\n",
    "            DecoderLayer(d_model=d_model, num_heads=num_heads, dff=dff, dropout_rate=dropout_rate)\n",
    "        for _ in range(num_layers)]\n",
    "\n",
    "        self.last_attn_scores = None\n",
    "\n",
    "    def call(self, x, context, *args, **kwargs):\n",
    "        x = self.pos_embedding(x)\n",
    "        x = self.dropout(x)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            x = self.dec_layers[i](x, context)\n",
    "\n",
    "        self.last_attn_scores = self.dec_layers[-1].last_attn_scores\n",
    "\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Transformer combined"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class Transformer(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, *, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, dropout_rate=0.1):\n",
    "        super().__init__()\n",
    "\n",
    "        self.encoder = Encoder(num_layers=num_layers, d_model=d_model, num_heads=num_heads, vocab_size=input_vocab_size, droput_rate=dropout_rate, dff=dff)\n",
    "\n",
    "        self.decoder = Decoder(num_layers=num_layers, d_model=d_model, num_heads=num_heads, dff=dff, vocab_size=target_vocab_size, dropout_rate=dropout_rate)\n",
    "\n",
    "        self.final_layer = tf.keras.layers.Dense(target_vocab_size)  # Why no Softmax??\n",
    "\n",
    "    def call(self, inputs, *args, **kwargs):\n",
    "        context, x = inputs\n",
    "\n",
    "        context = self.encoder(context)\n",
    "\n",
    "        x = self.decoder(x, context)\n",
    "\n",
    "        logits = self.final_layer(x)\n",
    "\n",
    "        try:\n",
    "            del logits._keras_mask\n",
    "        except AttributeError:\n",
    "            pass\n",
    "\n",
    "        return logits"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Custom Learning Rate Schedule"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "num_layers = 4\n",
    "d_model = 128\n",
    "dff = 512\n",
    "num_heads = 8\n",
    "dropout_rate = 0.1"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class CustomTransformerSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super().__init__()\n",
    "        self.d_model = tf.cast(d_model, tf.float32)\n",
    "        self.warmup_steps = warmup_steps\n",
    "\n",
    "    def __call__(self, step):\n",
    "        step = tf.cast(step, dtype=tf.float32)\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps ** -1.5)\n",
    "\n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "learning_rate = CustomTransformerSchedule(d_model)\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Custom Loss and Metric"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def masked_loss(label, pred):\n",
    "    mask = label != 0\n",
    "    loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "        from_logits=True, reduction='none'\n",
    "    )\n",
    "    loss = loss_obj(label, pred)\n",
    "    mask = tf.cast(mask, dtype=loss.dtype)\n",
    "    loss *= mask\n",
    "    loss = tf.reduce_sum(loss)/tf.reduce_sum(mask)\n",
    "    return loss"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def masked_accuracy(label, pred):\n",
    "    pred = tf.argmax(pred, axis=2)\n",
    "    label = tf.cast(label, dtype=pred.dtype)\n",
    "    match = label == pred\n",
    "\n",
    "    mask = label != 0\n",
    "    match = match & mask\n",
    "\n",
    "    match = tf.cast(match, dtype=tf.float32)\n",
    "    mask = tf.cast(mask, dtype=tf.float32)\n",
    "    return tf.reduce_sum(match)/tf.reduce_sum/mask"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Training"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "transformer = Transformer(\n",
    "    num_layers=num_layers,\n",
    "    d_model=d_model,\n",
    "    num_heads=num_heads,\n",
    "    dff=dff,\n",
    "    input_vocab_size=tokenizers.pt.get_vocab_size().numpy(),\n",
    "    target_vocab_size=tokenizers.en.get_vocab_size().numpy(),\n",
    "    dropout_rate=dropout_rate\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "transformer.compile(loss=masked_loss, optimizer=optimizer, metrics=[masked_accuracy])\n",
    "transformer.fit(train_batches, epochs=20, validation=val_batches)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Translating"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class Translator(tf.Module):\n",
    "\n",
    "    def __init__(self, tokenizers, transformer):\n",
    "        super().__init__()\n",
    "        self.tokenizers = tokenizers\n",
    "        self.transformer = transformer\n",
    "\n",
    "    def __call__(self, sentence, max_length=MAX_TOKENS):\n",
    "        assert isinstance(sentence, tf.Tensor)\n",
    "\n",
    "        if len(sentence.shape) == 0:\n",
    "            sentence = sentence[tf.newaxis]\n",
    "\n",
    "        sentence = self.tokenizers.pt.tokenize(sentence).to_tensor()\n",
    "\n",
    "        encoder_input = sentence\n",
    "        start_end = self.tokenizers.en.tokenize([''])[0]\n",
    "        start = start_end[0][tf.newaxis]\n",
    "        end = start_end[1][tf.newaxis]\n",
    "\n",
    "        output_array = tf.TensorArray(dtype=tf.int64, size=0, dynamic_size=True)\n",
    "        output_array.write(0, start)\n",
    "\n",
    "        for i in tf.range(max_length):\n",
    "            output = tf.transpose(output_array.stack())\n",
    "            predictions = self.transformer([encoder_input, output], training=False)\n",
    "\n",
    "            predictions = predictions[:, -1:, :]\n",
    "            predicted_id = tf.argmax(predictions, axis=-1)\n",
    "            output_array.write(i+1, predicted_id[0])\n",
    "\n",
    "            if predicted_id == end:\n",
    "                break\n",
    "\n",
    "        output = tf.transpose(output_array.stack())\n",
    "        text = tokenizers.en.detokenize(output)[0]\n",
    "        tokens = tokenizers.en.lookup(output)[0]\n",
    "\n",
    "        return text, tokens"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}