01_Regular_expressions_&_word_tokenization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "01. Regular expressions & word tokenization.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNEkmSo/WlaoxSq+z1u6D9q",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tanaymukherjee/Natural-Language-Processing/blob/master/01_Regular_expressions_%26_word_tokenization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D3y57KhBri76",
        "colab_type": "text"
      },
      "source": [
        "# Intro to Regex"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5p_vpN1xpajd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sHejjdyDpPdj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "my_string = \"Let's write RegEx!\""
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WU345PxbpmVg",
        "colab_type": "text"
      },
      "source": [
        "### Types of pattern macthing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dvRJpusEpXdu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "06c5dbc7-2df0-4fde-9938-b0a87c980f0e"
      },
      "source": [
        "# Split my_string on each space or punctuation\n",
        "print(re.findall(r\"\\w+\", my_string))\n",
        "\n",
        "# Split the whole string into each characters\n",
        "print(re.findall(r\"\\w\", my_string))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Let', 's', 'write', 'RegEx']\n",
            "['L', 'e', 't', 's', 'w', 'r', 'i', 't', 'e', 'R', 'e', 'g', 'E', 'x']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dXCEzjuqqFv1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "aad4447c-fe3f-4516-a96c-ba08ef6f64e4"
      },
      "source": [
        "# Write a pattern to match sentence endings: sentence_endings\n",
        "sentence_endings = r\"[.?!]\"\n",
        "\n",
        "# Split my_string on sentence endings and print the result\n",
        "print(re.split(sentence_endings, my_string))\n",
        "\n",
        "# Find all capitalized words in my_string and print the result\n",
        "capitalized_words = r\"[A-Z]\\w+\"\n",
        "print(re.findall(capitalized_words, my_string))\n",
        "\n",
        "# Split my_string on spaces and print the result\n",
        "spaces = r\"\\s+\"\n",
        "print(re.split(spaces, my_string))\n",
        "\n",
        "# Find all digits in my_string and print the result\n",
        "digits = r\"\\d+\"\n",
        "print(re.findall(digits, my_string))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[\"Let's write RegEx\", '']\n",
            "['Let', 'RegEx']\n",
            "[\"Let's\", 'write', 'RegEx!']\n",
            "[]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kmJEgED6rmb8",
        "colab_type": "text"
      },
      "source": [
        "# Intro to Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qAI_hylti3v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "2bfe0009-5e92-490e-9f44-88545a63f2b9"
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KdOoDOQorn1o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import necessary modules\n",
        "from nltk.tokenize import sent_tokenize\n",
        "from nltk.tokenize import word_tokenize"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k4p4t6u4sPqK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "scene_one = \"SCENE 1: [wind] [clop clop clop] \\nKING ARTHUR: Whoa there!  [clop clop clop] \\nSOLDIER #1: Halt!  Who goes there?\\nARTHUR: It is I, Arthur, son of Uther Pendragon, from the castle of Camelot.  King of the Britons, defeator of the Saxons, sovereign of all England!\\nSOLDIER #1: Pull the other one!\\nARTHUR: I am, ...  and this is my trusty servant Patsy.  We have ridden the length and breadth of the land in search of knights who will join me in my court at Camelot.  I must speak with your lord and master.\\nSOLDIER #1: What?  Ridden on a horse?\\nARTHUR: Yes!\\nSOLDIER #1: You're using coconuts!\\nARTHUR: What?\\nSOLDIER #1: You've got two empty halves of coconut and you're bangin' 'em together.\\nARTHUR: So?  We have ridden since the snows of winter covered this land, through the kingdom of Mercea, through--\\nSOLDIER #1: Where'd you get the coconuts?\\nARTHUR: We found them.\\nSOLDIER #1: Found them?  In Mercea?  The coconut's tropical!\\nARTHUR: What do you mean?\\nSOLDIER #1: Well, this is a temperate zone.\\nARTHUR: The swallow may fly south with the sun or the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land?\\nSOLDIER #1: Are you suggesting coconuts migrate?\\nARTHUR: Not at all.  They could be carried.\\nSOLDIER #1: What?  A swallow carrying a coconut?\\nARTHUR: It could grip it by the husk!\\nSOLDIER #1: It's not a question of where he grips it!  It's a simple question of weight ratios!  A five ounce bird could not carry a one pound coconut.\\nARTHUR: Well, it doesn't matter.  Will you go and tell your master that Arthur from the Court of Camelot is here.\\nSOLDIER #1: Listen.  In order to maintain air-speed velocity, a swallow needs to beat its wings forty-three times every second, right?\\nARTHUR: Please!\\nSOLDIER #1: Am I right?\\nARTHUR: I'm not interested!\\nSOLDIER #2: It could be carried by an African swallow!\\nSOLDIER #1: Oh, yeah, an African swallow maybe, but not a European swallow.  That's my point.\\nSOLDIER #2: Oh, yeah, I agree with that.\\nARTHUR: Will you ask your master if he wants to join my court at Camelot?!\\nSOLDIER #1: But then of course a-- African swallows are non-migratory.\\nSOLDIER #2: Oh, yeah...\\nSOLDIER #1: So they couldn't bring a coconut back anyway...  [clop clop clop] \\nSOLDIER #2: Wait a minute!  Supposing two swallows carried it together?\\nSOLDIER #1: No, they'd have to have it on a line.\\nSOLDIER #2: Well, simple!  They'd just use a strand of creeper!\\nSOLDIER #1: What, held under the dorsal guiding feathers?\\nSOLDIER #2: Well, why not?\\n\""
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jNhKt_tzsTbA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 759
        },
        "outputId": "fda4580b-52b4-496d-eb5b-e2803fb2a4a6"
      },
      "source": [
        "print(scene_one)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SCENE 1: [wind] [clop clop clop] \n",
            "KING ARTHUR: Whoa there!  [clop clop clop] \n",
            "SOLDIER #1: Halt!  Who goes there?\n",
            "ARTHUR: It is I, Arthur, son of Uther Pendragon, from the castle of Camelot.  King of the Britons, defeator of the Saxons, sovereign of all England!\n",
            "SOLDIER #1: Pull the other one!\n",
            "ARTHUR: I am, ...  and this is my trusty servant Patsy.  We have ridden the length and breadth of the land in search of knights who will join me in my court at Camelot.  I must speak with your lord and master.\n",
            "SOLDIER #1: What?  Ridden on a horse?\n",
            "ARTHUR: Yes!\n",
            "SOLDIER #1: You're using coconuts!\n",
            "ARTHUR: What?\n",
            "SOLDIER #1: You've got two empty halves of coconut and you're bangin' 'em together.\n",
            "ARTHUR: So?  We have ridden since the snows of winter covered this land, through the kingdom of Mercea, through--\n",
            "SOLDIER #1: Where'd you get the coconuts?\n",
            "ARTHUR: We found them.\n",
            "SOLDIER #1: Found them?  In Mercea?  The coconut's tropical!\n",
            "ARTHUR: What do you mean?\n",
            "SOLDIER #1: Well, this is a temperate zone.\n",
            "ARTHUR: The swallow may fly south with the sun or the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land?\n",
            "SOLDIER #1: Are you suggesting coconuts migrate?\n",
            "ARTHUR: Not at all.  They could be carried.\n",
            "SOLDIER #1: What?  A swallow carrying a coconut?\n",
            "ARTHUR: It could grip it by the husk!\n",
            "SOLDIER #1: It's not a question of where he grips it!  It's a simple question of weight ratios!  A five ounce bird could not carry a one pound coconut.\n",
            "ARTHUR: Well, it doesn't matter.  Will you go and tell your master that Arthur from the Court of Camelot is here.\n",
            "SOLDIER #1: Listen.  In order to maintain air-speed velocity, a swallow needs to beat its wings forty-three times every second, right?\n",
            "ARTHUR: Please!\n",
            "SOLDIER #1: Am I right?\n",
            "ARTHUR: I'm not interested!\n",
            "SOLDIER #2: It could be carried by an African swallow!\n",
            "SOLDIER #1: Oh, yeah, an African swallow maybe, but not a European swallow.  That's my point.\n",
            "SOLDIER #2: Oh, yeah, I agree with that.\n",
            "ARTHUR: Will you ask your master if he wants to join my court at Camelot?!\n",
            "SOLDIER #1: But then of course a-- African swallows are non-migratory.\n",
            "SOLDIER #2: Oh, yeah...\n",
            "SOLDIER #1: So they couldn't bring a coconut back anyway...  [clop clop clop] \n",
            "SOLDIER #2: Wait a minute!  Supposing two swallows carried it together?\n",
            "SOLDIER #1: No, they'd have to have it on a line.\n",
            "SOLDIER #2: Well, simple!  They'd just use a strand of creeper!\n",
            "SOLDIER #1: What, held under the dorsal guiding feathers?\n",
            "SOLDIER #2: Well, why not?\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PyzjPbRHr3tP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "91c6f7a6-abf5-4ea7-a8d9-d430ef29403f"
      },
      "source": [
        "# Split scene_one into sentences: sentences\n",
        "sentences = sent_tokenize(scene_one)\n",
        "print(sentences)\n",
        "\n",
        "# Use word_tokenize to tokenize the fourth sentence: tokenized_sent\n",
        "tokenized_sent = word_tokenize(sentences[3])\n",
        "\n",
        "# Make a set of unique tokens in the entire scene: unique_tokens\n",
        "unique_tokens = set(word_tokenize(scene_one))\n",
        "\n",
        "# Print the unique tokens result\n",
        "print(unique_tokens)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['SCENE 1: [wind] [clop clop clop] \\nKING ARTHUR: Whoa there!', '[clop clop clop] \\nSOLDIER #1: Halt!', 'Who goes there?', 'ARTHUR: It is I, Arthur, son of Uther Pendragon, from the castle of Camelot.', 'King of the Britons, defeator of the Saxons, sovereign of all England!', 'SOLDIER #1: Pull the other one!', 'ARTHUR: I am, ...  and this is my trusty servant Patsy.', 'We have ridden the length and breadth of the land in search of knights who will join me in my court at Camelot.', 'I must speak with your lord and master.', 'SOLDIER #1: What?', 'Ridden on a horse?', 'ARTHUR: Yes!', \"SOLDIER #1: You're using coconuts!\", 'ARTHUR: What?', \"SOLDIER #1: You've got two empty halves of coconut and you're bangin' 'em together.\", 'ARTHUR: So?', \"We have ridden since the snows of winter covered this land, through the kingdom of Mercea, through--\\nSOLDIER #1: Where'd you get the coconuts?\", 'ARTHUR: We found them.', 'SOLDIER #1: Found them?', 'In Mercea?', \"The coconut's tropical!\", 'ARTHUR: What do you mean?', 'SOLDIER #1: Well, this is a temperate zone.', 'ARTHUR: The swallow may fly south with the sun or the house martin or the plover may seek warmer climes in winter, yet these are not strangers to our land?', 'SOLDIER #1: Are you suggesting coconuts migrate?', 'ARTHUR: Not at all.', 'They could be carried.', 'SOLDIER #1: What?', 'A swallow carrying a coconut?', 'ARTHUR: It could grip it by the husk!', \"SOLDIER #1: It's not a question of where he grips it!\", \"It's a simple question of weight ratios!\", 'A five ounce bird could not carry a one pound coconut.', \"ARTHUR: Well, it doesn't matter.\", 'Will you go and tell your master that Arthur from the Court of Camelot is here.', 'SOLDIER #1: Listen.', 'In order to maintain air-speed velocity, a swallow needs to beat its wings forty-three times every second, right?', 'ARTHUR: Please!', 'SOLDIER #1: Am I right?', \"ARTHUR: I'm not interested!\", 'SOLDIER #2: It could be carried by an African swallow!', 'SOLDIER #1: Oh, yeah, an African swallow maybe, but not a European swallow.', \"That's my point.\", 'SOLDIER #2: Oh, yeah, I agree with that.', 'ARTHUR: Will you ask your master if he wants to join my court at Camelot?!', 'SOLDIER #1: But then of course a-- African swallows are non-migratory.', 'SOLDIER #2: Oh, yeah...', \"SOLDIER #1: So they couldn't bring a coconut back anyway...  [clop clop clop] \\nSOLDIER #2: Wait a minute!\", 'Supposing two swallows carried it together?', \"SOLDIER #1: No, they'd have to have it on a line.\", 'SOLDIER #2: Well, simple!', \"They'd just use a strand of creeper!\", 'SOLDIER #1: What, held under the dorsal guiding feathers?', 'SOLDIER #2: Well, why not?']\n",
            "{'why', 'ridden', 'sovereign', 'Supposing', 'anyway', \"n't\", 'point', 'maybe', 'seek', 'carried', 'migrate', \"'m\", 'just', 'It', 'with', 'sun', 'he', ']', 'halves', 'bird', 'son', '--', 'forty-three', 'creeper', 'may', 'length', 'pound', 'my', 'Mercea', 'Whoa', 'to', 'SOLDIER', 'a', 'must', '.', 'needs', 'Where', 'then', 'where', 'right', 'King', 'maintain', 'Pull', 'found', '!', 'swallow', 'and', 'who', 'suggesting', 'all', 'fly', 'carry', 'warmer', 'are', 'tropical', 'They', 'zone', 'Saxons', 'speak', 'join', 'since', 'the', 'Pendragon', 'have', 'The', 'KING', 'use', '...', 'We', 'weight', 'this', 'Not', \"'\", 'ounce', 'times', 'What', 'under', 'five', 'together', 'grips', 'Court', 'on', 'using', 'beat', 'question', 'that', 'your', 'by', 'wants', '[', 'England', 'dorsal', 'Who', 'other', 'search', 'Please', 'Are', 'temperate', 'servant', 'Wait', 'there', 'European', 'goes', '#', 'Uther', 'here', 'them', 'clop', 'one', 'You', 'Camelot', \"'em\", 'That', 'got', 'breadth', 'covered', 'martin', 'second', 'trusty', 'non-migratory', 'get', 'order', 'simple', 'every', 'bring', 'strand', 'will', 'castle', 'minute', 'agree', 'A', 'does', 'but', 'ratios', 'wings', 'plover', '1', 'interested', 'they', 'or', 'in', 'swallows', 'air-speed', 'carrying', 'But', 'coconuts', 'Am', 'you', '2', 'is', 'am', 'these', \"'re\", 'Oh', 'go', 'Found', 'house', 'ask', 'course', 'African', 'its', 'matter', ',', \"'s\", 'at', 'Listen', 'if', 'No', 'defeator', \"'d\", 'wind', 'two', 'SCENE', 'climes', 'yet', 'our', 'line', 'snows', ':', 'an', 'do', 'husk', 'lord', 'yeah', 'grip', 'guiding', \"'ve\", 'Ridden', 'I', 'me', 'Arthur', 'empty', 'Britons', 'In', 'So', 'be', 'it', 'held', 'bangin', 'mean', 'court', 'from', 'of', 'not', 'Yes', '?', 'Well', 'strangers', 'velocity', 'horse', 'feathers', 'Will', 'back', 'knights', 'winter', 'tell', 'through', 'could', 'south', 'land', 'coconut', 'ARTHUR', 'Patsy', 'Halt', 'kingdom', 'master'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jv01YI_sttsL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "50ed5c32-d653-458a-be77-30526a701d8f"
      },
      "source": [
        "# Search for the first occurrence of \"coconuts\" in scene_one: match\n",
        "match = re.search(\"coconuts\", scene_one)\n",
        "\n",
        "# Print the start and end indexes of match\n",
        "print(match.start(), match.end())"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "580 588\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kEmTN7vUJPDt",
        "colab_type": "text"
      },
      "source": [
        "## Regex using tokenization package"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JOnnm9j3JCLH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.tokenize import regexp_tokenize"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BxWKIQMIIhlF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "my_string = \"SOLDIER #1: Found them? In Mercea? The coconut's tropical!\"\n",
        "\n",
        "pattern1 = r\"(\\w+|\\?|!)\"\n",
        "pattern2 = r\"(\\w+|#\\d|\\?|!)\"\n",
        "pattern3 = r\"(#\\d\\w+\\?!)\"\n",
        "pattern4 = r\"\\s+\""
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zeRSwUxJJy_y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246
        },
        "outputId": "7c853e70-398f-4641-d718-811a8e614e87"
      },
      "source": [
        "regexp_tokenize(my_string, pattern1)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['SOLDIER',\n",
              " '1',\n",
              " 'Found',\n",
              " 'them',\n",
              " '?',\n",
              " 'In',\n",
              " 'Mercea',\n",
              " '?',\n",
              " 'The',\n",
              " 'coconut',\n",
              " 's',\n",
              " 'tropical',\n",
              " '!']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbWFenzCJzF1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246
        },
        "outputId": "71b86e41-b2cf-4fc1-9541-701d47369470"
      },
      "source": [
        "regexp_tokenize(my_string, pattern2)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['SOLDIER',\n",
              " '#1',\n",
              " 'Found',\n",
              " 'them',\n",
              " '?',\n",
              " 'In',\n",
              " 'Mercea',\n",
              " '?',\n",
              " 'The',\n",
              " 'coconut',\n",
              " 's',\n",
              " 'tropical',\n",
              " '!']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NJ3d06BmJzLy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "139b9a24-a537-4c93-b399-e2a6132c1e18"
      },
      "source": [
        "regexp_tokenize(my_string, pattern3)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VkFb273bJ3ig",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "85836106-e469-40e9-8de5-a7dcda5307d9"
      },
      "source": [
        "regexp_tokenize(my_string, pattern4)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h0h-DHtJLAiV",
        "colab_type": "text"
      },
      "source": [
        "### Non-ascii tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zMCQBdVtLBp7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import the necessary modules\n",
        "from nltk.tokenize import regexp_tokenize\n",
        "from nltk.tokenize import TweetTokenizer"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3tWKVJxYLFg2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "german_text = \"Wann gehen wir Pizza essen? 🍕 Und fährst du mit Über? 🚕\""
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RpGdo-A-LI0t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "1f74e4e6-cd1b-49d1-d04f-bbd31f7f7682"
      },
      "source": [
        "# Tokenize and print all words in german_text\n",
        "all_words = word_tokenize(german_text)\n",
        "print(all_words)\n",
        "\n",
        "# Tokenize and print only capital words\n",
        "capital_words = r\"[A-ZÜ]\\w+\"\n",
        "print(regexp_tokenize(german_text, capital_words))\n",
        "\n",
        "# Tokenize and print only emoji\n",
        "emoji = \"['\\U0001F300-\\U0001F5FF'|'\\U0001F600-\\U0001F64F'|'\\U0001F680-\\U0001F6FF'|'\\u2600-\\u26FF\\u2700-\\u27BF']\"\n",
        "print(regexp_tokenize(german_text, emoji))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Wann', 'gehen', 'wir', 'Pizza', 'essen', '?', '🍕', 'Und', 'fährst', 'du', 'mit', 'Über', '?', '🚕']\n",
            "['Wann', 'Pizza', 'Und', 'Über']\n",
            "['🍕', '🚕']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x3-8N5PkMIHm",
        "colab_type": "text"
      },
      "source": [
        "### Charting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i5awMp9QMgs8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from matplotlib import pyplot as plt"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gUO-hQ4eMTAp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "23c57122-abfd-4a14-9783-d1482f0e2248"
      },
      "source": [
        "# Split the script into lines: lines\n",
        "lines = scene_one.split('\\n')\n",
        "\n",
        "# Replace all script lines for speaker\n",
        "pattern = \"[A-Z]{2,}(\\s)?(#\\d)?([A-Z]{2,})?:\"\n",
        "lines = [re.sub(pattern, '', l) for l in lines]\n",
        "\n",
        "# Tokenize each line: tokenized_lines\n",
        "tokenized_lines = [regexp_tokenize(s, \"\\w+\") for s in lines]\n",
        "\n",
        "# Make a frequency list of lengths: line_num_words\n",
        "line_num_words = [len(t_line) for t_line in tokenized_lines]\n",
        "\n",
        "# Plot a histogram of the line lengths\n",
        "plt.hist(line_num_words)\n",
        "\n",
        "# Show the plot\n",
        "plt.show()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOjklEQVR4nO3dfYxldX3H8fenLKgoKdKdUuoyHTRKY40PZGq1WqugZgtGbOIfkNpgSzKJqRZbLV1qUu0fJmitD0kbzVZWSKVYi1iNpC1UMaSJRXdxgYVF8WGrS9FdQowPbUXqt3/cQzuOs3Pv3Htmzvzw/Uomc865Z+b32V9mPnvm3HvuSVUhSWrPTw0dQJI0HQtckhplgUtSoyxwSWqUBS5Jjdq2mYNt3769FhYWNnNISWrevn377q+quZXbN7XAFxYW2Lt372YOKUnNS/Lvq233FIokNcoCl6RGWeCS1CgLXJIaZYFLUqMscElq1NgCT7InyZEkB1Zsf12Su5PcmeTtGxdRkrSaSY7ArwR2Lt+Q5EXA+cAzquqXgHf0H02StJaxBV5VNwMPrNj8GuDyqvp+t8+RDcgmSVrDtFdiPgX4tSRvBf4beGNVfW61HZMsAUsA8/PzUw43rIVd1w8y7qHLzxtkXEltmPZJzG3AKcBzgD8CPpwkq+1YVburarGqFufmfuxSfknSlKYt8MPAdTXyWeCHwPb+YkmSxpm2wP8BeBFAkqcAJwD39xVKkjTe2HPgSa4BXghsT3IYeDOwB9jTvbTwQeCi8u7IkrSpxhZ4VV14jIde1XMWSdI6eCWmJDXKApekRlngktQoC1ySGmWBS1KjLHBJapQFLkmNssAlqVEWuCQ1ygKXpEZZ4JLUKAtckhplgUtSoyxwSWqUBS5JjbLAJalRYws8yZ4kR7q776x87A1JKon3w5SkTTbJEfiVwM6VG5OcDrwU+FrPmSRJExhb4FV1M/DAKg+9C7gU8F6YkjSAqc6BJzkfuLeqbus5jyRpQmNvarxSkhOBP2F0+mSS/ZeAJYD5+fn1DidJOoZpjsCfBJwB3JbkELADuDXJz622c1XtrqrFqlqcm5ubPqkk6Ues+wi8qu4Afvbh9a7EF6vq/h5zSZLGmORlhNcAnwHOTHI4ycUbH0uSNM7YI/CqunDM4wu9pZEkTcwrMSWpURa4JDXKApekRlngktQoC1ySGmWBS1KjLHBJapQFLkmNssAlqVEWuCQ1ygKXpEZZ4JLUKAtckhplgUtSoyxwSWqUBS5JjbLAJalRk9xSbU+SI0kOLNv250nuTnJ7ko8mOXljY0qSVprkCPxKYOeKbTcCT6uqpwNfBC7rOZckaYyxBV5VNwMPrNh2Q1U91K3+G7BjA7JJktbQxznw3wX+8VgPJllKsjfJ3qNHj/YwnCQJZizwJG8CHgKuPtY+VbW7qharanFubm6W4SRJy2yb9guTvBp4GXBOVVVviSRJE5mqwJPsBC4Ffr2q/rPfSJKkSUzyMsJrgM8AZyY5nORi4C+Bk4Abk+xP8r4NzilJWmHsEXhVXbjK5is2IIskaR28ElOSGmWBS1KjLHBJapQFLkmNssAlqVEWuCQ1ygKXpEZZ4JLUKAtckhplgUtSoyxwSWqUBS5JjbLAJalRFrgkNcoCl6RGWeCS1CgLXJIaNckt1fYkOZLkwLJtpyS5Mck93efHb2xMSdJKkxyBXwnsXLFtF/DJqnoy8MluXZK0icYWeFXdDDywYvP5wFXd8lXAK3rOJUkaY9pz4KdW1X3d8jeAU4+1Y5KlJHuT7D169OiUw0mSVpr5ScyqKqDWeHx3VS1W1eLc3Nysw0mSOtMW+DeTnAbQfT7SXyRJ0iSmLfCPAxd1yxcBH+snjiRpUpO8jPAa4DPAmUkOJ7kYuBx4SZJ7gBd365KkTbRt3A5VdeExHjqn5yySpHXwSkxJapQFLkmNssAlqVEWuCQ1ygKXpEZZ4JLUKAtckhplgUtSoyxwSWqUBS5JjbLAJalRFrgkNcoCl6RGWeCS1CgLXJIaZYFLUqMscElq1EwFnuQPktyZ5ECSa5I8uq9gkqS1TV3gSZ4A/D6wWFVPA44DLugrmCRpbbOeQtkGPCbJNuBE4D9mjyRJmsTYmxofS1Xdm+QdwNeA/wJuqKobVu6XZAlYApifn592OBZ2XT/110rSI9Esp1AeD5wPnAH8PPDYJK9auV9V7a6qxapanJubmz6pJOlHzHIK5cXAV6vqaFX9ALgO+NV+YkmSxpmlwL8GPCfJiUkCnAMc7CeWJGmcqQu8qm4BrgVuBe7ovtfunnJJksaY+klMgKp6M/DmnrJIktbBKzElqVEWuCQ1ygKXpEZZ4JLUKAtckhplgUtSoyxwSWqUBS5JjbLAJalRFrgkNcoCl6RGWeCS1CgLXJIaZYFLUqMscElqlAUuSY2ywCWpUTMVeJKTk1yb5O4kB5M8t69gkqS1zXRLNeA9wD9V1SuTnACc2EMmSdIEpi7wJD8NvAB4NUBVPQg82E8sSdI4sxyBnwEcBT6Q5BnAPuCSqvre8p2SLAFLAPPz8zMM95NnYdf1g4x76PLzBhlX0vrMcg58G3AW8N6qehbwPWDXyp2qandVLVbV4tzc3AzDSZKWm6XADwOHq+qWbv1aRoUuSdoEUxd4VX0D+HqSM7tN5wB39ZJKkjTWrK9CeR1wdfcKlK8AvzN7JEnSJGYq8KraDyz2lEWStA5eiSlJjbLAJalRFrgkNcoCl6RGWeCS1CgLXJIaZYFLUqMscElqlAUuSY2ywCWpURa4JDXKApekRlngktQoC1ySGmWBS1KjLHBJapQFLkmNmrnAkxyX5PNJPtFHIEnSZPo4Ar8EONjD95EkrcNMBZ5kB3Ae8P5+4kiSJjXrXenfDVwKnHSsHZIsAUsA8/PzMw6nzbCw6/rBxj50+XmDjS21Zuoj8CQvA45U1b619quq3VW1WFWLc3Nz0w4nSVphllMozwNenuQQ8CHg7CQf7CWVJGmsqQu8qi6rqh1VtQBcAHyqql7VWzJJ0pp8HbgkNWrWJzEBqKpPA5/u43tJkibjEbgkNcoCl6RGWeCS1CgLXJIaZYFLUqMscElqlAUuSY2ywCWpURa4JDXKApekRlngktQoC1ySGmWBS1KjLHBJapQFLkmNssAlqVEWuCQ1apa70p+e5KYkdyW5M8klfQaTJK1tlluqPQS8oapuTXISsC/JjVV1V0/ZJElrmOWu9PdV1a3d8neAg8AT+gomSVpbLzc1TrIAPAu4ZZXHloAlgPn5+T6Gk3q3sOv6wcY+dPl5g4z7k/hvfqSZ+UnMJI8DPgK8vqq+vfLxqtpdVYtVtTg3NzfrcJKkzkwFnuR4RuV9dVVd108kSdIkZnkVSoArgINV9c7+IkmSJjHLEfjzgN8Gzk6yv/s4t6dckqQxpn4Ss6r+FUiPWSRJ6+CVmJLUKAtckhplgUtSoyxwSWqUBS5JjbLAJalRFrgkNaqXN7OS+jLkGywN5Sfx3zyUR9obeHkELkmNssAlqVEWuCQ1ygKXpEZZ4JLUKAtckhplgUtSoyxwSWqUBS5JjZr1psY7k3whyZeS7OorlCRpvFluanwc8FfAbwBPBS5M8tS+gkmS1jbLEfizgS9V1Veq6kHgQ8D5/cSSJI0zy5tZPQH4+rL1w8CvrNwpyRKw1K1+N8kXphxvO3D/lF+7kcy1PuZan0dkrrytxyQ/aqvOF3nbTNl+YbWNG/5uhFW1G9g96/dJsreqFnuI1CtzrY+51sdc67NVc8HGZJvlFMq9wOnL1nd02yRJm2CWAv8c8OQkZyQ5AbgA+Hg/sSRJ40x9CqWqHkryWuCfgeOAPVV1Z2/JftzMp2E2iLnWx1zrY6712aq5YAOypar6/p6SpE3glZiS1CgLXJIa1USBb9VL9pMcSnJHkv1J9g6YY0+SI0kOLNt2SpIbk9zTfX78Fsn1liT3dnO2P8m5A+Q6PclNSe5KcmeSS7rtg87ZGrkGnbMkj07y2SS3dbn+rNt+RpJbut/Lv+tezLAVcl2Z5KvL5uuZm5lrWb7jknw+ySe69f7nq6q29AejJ0i/DDwROAG4DXjq0Lm6bIeA7VsgxwuAs4ADy7a9HdjVLe8C3rZFcr0FeOPA83UacFa3fBLwRUZvBzHonK2Ra9A5AwI8rls+HrgFeA7wYeCCbvv7gNdskVxXAq8c8mesy/SHwN8Cn+jWe5+vFo7AvWR/jKq6GXhgxebzgau65auAV2xqKI6Za3BVdV9V3dotfwc4yOjK4kHnbI1cg6qR73arx3cfBZwNXNttH2K+jpVrcEl2AOcB7+/WwwbMVwsFvtol+4P/UHcKuCHJvu4tA7aSU6vqvm75G8CpQ4ZZ4bVJbu9OsWz6qZ3lkiwAz2J09LZl5mxFLhh4zrrTAfuBI8CNjP4q/lZVPdTtMsjv5cpcVfXwfL21m693JXnUZucC3g1cCvywW/8ZNmC+Wijwrez5VXUWo3dk/L0kLxg60Gpq9DfbljgyAd4LPAl4JnAf8BdDBUnyOOAjwOur6tvLHxtyzlbJNficVdX/VNUzGV1x/WzgFzc7w2pW5kryNOAyRvl+GTgF+OPNzJTkZcCRqtq30WO1UOBb9pL9qrq3+3wE+CijH+yt4ptJTgPoPh8ZOA8AVfXN7pfuh8BfM9CcJTmeUUleXVXXdZsHn7PVcm2VOeuyfAu4CXgucHKShy8GHPT3clmund2pqKqq7wMfYPPn63nAy5McYnTK92zgPWzAfLVQ4Fvykv0kj01y0sPLwEuBA2t/1ab6OHBRt3wR8LEBs/yfhwuy85sMMGfd+cgrgINV9c5lDw06Z8fKNfScJZlLcnK3/BjgJYzOz98EvLLbbYj5Wi3X3cv+Ew6j88ybOl9VdVlV7aiqBUZ99amq+i02Yr6GfqZ2wmdzz2X0jPyXgTcNnafL9ERGr4i5DbhzyFzANYz+tP4Bo3NrFzM65/ZJ4B7gX4BTtkiuvwHuAG5nVJinDZDr+YxOj9wO7O8+zh16ztbINeicAU8HPt+NfwD40277E4HPAl8C/h541BbJ9aluvg4AH6R7pcoQH8AL+f9XofQ+X15KL0mNauEUiiRpFRa4JDXKApekRlngktQoC1ySGmWBS1KjLHBJatT/AuvZ2tOUOCMyAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}