indicTrans_hosted_api_inference.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "indicTrans_hosted_api_inference.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPGTNpiMN7bQWok+EhrXX3e",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AI4Bharat/indicTrans/blob/main/indicTrans_hosted_api_inference.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install mosestokenizer\n",
        "!pip install indic-nlp-library"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w3MIksjntRle",
        "outputId": "5beb9ccd-c0c1-49f3-cc07-f115dd9ecbf2"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting mosestokenizer\n",
            "  Downloading mosestokenizer-1.2.1.tar.gz (37 kB)\n",
            "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from mosestokenizer) (0.6.2)\n",
            "Collecting openfile\n",
            "  Downloading openfile-0.0.7-py3-none-any.whl (2.4 kB)\n",
            "Collecting uctools\n",
            "  Downloading uctools-1.3.0.tar.gz (4.6 kB)\n",
            "Collecting toolwrapper\n",
            "  Downloading toolwrapper-2.1.0.tar.gz (3.2 kB)\n",
            "Building wheels for collected packages: mosestokenizer, toolwrapper, uctools\n",
            "  Building wheel for mosestokenizer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mosestokenizer: filename=mosestokenizer-1.2.1-py3-none-any.whl size=49189 sha256=92b4b8f7ff5794912fa5888562f1408ea92d51c87654caf0e56bddbb5b47c665\n",
            "  Stored in directory: /root/.cache/pip/wheels/b0/35/f7/af1258779a0b890abc3c79481460c597cb1f3659d0603cfb9d\n",
            "  Building wheel for toolwrapper (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for toolwrapper: filename=toolwrapper-2.1.0-py3-none-any.whl size=3353 sha256=332430df7c639a99e2584df60d741c4b692620e573a413ba0ff76ab57997296c\n",
            "  Stored in directory: /root/.cache/pip/wheels/c5/4f/33/54741ffe08e38ececb1d28068a153729b4fe820bafa0a0691f\n",
            "  Building wheel for uctools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for uctools: filename=uctools-1.3.0-py3-none-any.whl size=6161 sha256=7bbef7aa00f076b2c5045f24c83ec7b0de9a64cba16eecfff46a5dc728520900\n",
            "  Stored in directory: /root/.cache/pip/wheels/fb/44/e9/914cf8fa71f0141f9314f862538d1218fcf2b94542a0fb7d35\n",
            "Successfully built mosestokenizer toolwrapper uctools\n",
            "Installing collected packages: uctools, toolwrapper, openfile, mosestokenizer\n",
            "Successfully installed mosestokenizer-1.2.1 openfile-0.0.7 toolwrapper-2.1.0 uctools-1.3.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting indic-nlp-library\n",
            "  Downloading indic_nlp_library-0.81-py3-none-any.whl (40 kB)\n",
            "\u001b[K     |████████████████████████████████| 40 kB 3.0 MB/s \n",
            "\u001b[?25hCollecting sphinx-rtd-theme\n",
            "  Downloading sphinx_rtd_theme-1.0.0-py2.py3-none-any.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 7.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from indic-nlp-library) (1.21.6)\n",
            "Collecting sphinx-argparse\n",
            "  Downloading sphinx_argparse-0.3.1-py2.py3-none-any.whl (12 kB)\n",
            "Collecting morfessor\n",
            "  Downloading Morfessor-2.0.6-py3-none-any.whl (35 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from indic-nlp-library) (1.3.5)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->indic-nlp-library) (2022.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->indic-nlp-library) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->indic-nlp-library) (1.15.0)\n",
            "Requirement already satisfied: sphinx>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx-argparse->indic-nlp-library) (1.8.6)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.2.0)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.10.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (57.4.0)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.7.12)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.4)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (21.3)\n",
            "Requirement already satisfied: docutils<0.18,>=0.11 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.17.1)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.3.0)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.6.1)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.11.3)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.0.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2022.6.15)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (3.0.9)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-websupport->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.1.5)\n",
            "Installing collected packages: sphinx-rtd-theme, sphinx-argparse, morfessor, indic-nlp-library\n",
            "Successfully installed indic-nlp-library-0.81 morfessor-2.0.6 sphinx-argparse-0.3.1 sphinx-rtd-theme-1.0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "0LVUe-Ubn0HZ"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "import json\n",
        "from pprint import pprint"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "uri = \"http://216.48.181.177:5050\""
      ],
      "metadata": {
        "id": "1vyIgPAbJUZj"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# to translate one sentence, we use the translate sentence endpoint\n",
        "\n",
        "API_URL = f\"{uri}/translate_sentence\"\n",
        "\n",
        "\n",
        "# In the Json field of the request, we specify the text we want translation for, the source language and the target language\n",
        "\n",
        "response = requests.post(\n",
        "    API_URL,\n",
        "    json={\n",
        "  \"text\": \"If you develop these symptoms in someone close to you, staying at home can help prevent the spread of Coronavirus infection.\",\n",
        "  \"source_language\": \"en\",\n",
        "  \"target_language\": \"ta\"\n",
        "},\n",
        ")"
      ],
      "metadata": {
        "id": "DofbwJkfoNLn"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output = json.loads(response.text)\n",
        "print(output)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ycBDU9UToqV5",
        "outputId": "f9a6d2a6-e23c-4ed7-9eea-6cbe6df0e7e2"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'text': 'உங்களுக்கு நெருக்கமான ஒருவரிடம் இந்த அறிகுறிகள் உங்களுக்கு ஏற்பட்டால், வீட்டிலேயே தங்கியிருப்பது கொரோனா வைரஸ் தொற்று பரவுவதைத் தடுக்க உதவும்.', 'duration': 0.85}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Request completed in {output['duration']} seconds and the translation is {output['text']}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e6T0XhrEorbf",
        "outputId": "e4d532b2-5eca-43d8-c521-7a872933f859"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Request completed in 0.85 seconds and the translation is உங்களுக்கு நெருக்கமான ஒருவரிடம் இந்த அறிகுறிகள் உங்களுக்கு ஏற்பட்டால், வீட்டிலேயே தங்கியிருப்பது கொரோனா வைரஸ் தொற்று பரவுவதைத் தடுக்க உதவும்.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# to translate a batch of sentences, we use batch_translate endpoint\n",
        "API_URL = f\"{uri}/batch_translate\"\n",
        "\n",
        "# This is a sample batch of 4 tamil sentences. IF you have a large batch of sentences, please break it into smaller batches (typically of size 8 or 16) and query the API multiple times.\n",
        "\n",
        "sentence_batch = ['இத்தொற்றுநோய் உலகளாவிய சமூக மற்றும் பொருளாதார சீர்குலைவை ஏற்படுத்தியுள்ளது.',\n",
        " 'இதனால் பெரும் பொருளாதார மந்தநிலைக்குப் பின்னர் உலகளவில் மிகப்பெரிய மந்தநிலை ஏற்பட்டுள்ளது.',\n",
        " 'இது விளையாட்டு,மத, அரசியல் மற்றும் கலாச்சார நிகழ்வுகளை ஒத்திவைக்க அல்லது ரத்து செய்ய வழிவகுத்தது.',\n",
        " 'அச்சம் காரணமாக முகக்கவசம், கிருமிநாசினி உள்ளிட்ட பொருட்களை அதிக நபர்கள் வாங்கியதால் விநியோகப் பற்றாக்குறை ஏற்பட்டது.']\n",
        "\n",
        "# here we give the sentence_batch to \"text_lines\" and change the source and target language accordingly\n",
        "response = requests.post(\n",
        "    API_URL,\n",
        "    json={\n",
        "  \"text_lines\": sentence_batch,\n",
        "  \"source_language\": \"ta\",\n",
        "  \"target_language\": \"en\"\n",
        "},\n",
        ")\n",
        "\n",
        "output = json.loads(response.text)"
      ],
      "metadata": {
        "id": "4ddhHhrlqAFt"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(output)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MlP5ukzQr651",
        "outputId": "5caacf4d-af11-4eb8-c5dd-3d622f07fde2"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'text_lines': ['The pandemic has caused global social and economic disruption.', 'This has led to the worlds largest recession since the Great Depression.', 'This led to the postponement or cancellation of sporting, religious, political and cultural events.', 'Due to this fear, there was a shortage of supply as most of the people purchased the items like masks, sanitizers etc.'], 'duration': 0.34}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Request completed in {output['duration']} seconds and the translation is {output['text_lines']}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XaxIEMIertV-",
        "outputId": "cd611735-f0e2-42a7-eeea-12647b9ad125"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Request completed in 0.34 seconds and the translation is ['The pandemic has caused global social and economic disruption.', 'This has led to the worlds largest recession since the Great Depression.', 'This led to the postponement or cancellation of sporting, religious, political and cultural events.', 'Due to this fear, there was a shortage of supply as most of the people purchased the items like masks, sanitizers etc.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# if you want to translate a paragraph, you can first break it into sentences first\n",
        "\n",
        "# the following code shows you how to break the paragraph into batch of sentences\n",
        "\n",
        "# install these libraries\n",
        "# pip install mosestokenizer\n",
        "# pip install indic-nlp-library\n",
        "\n",
        "from mosestokenizer import *\n",
        "from indicnlp.tokenize import sentence_tokenize\n",
        "\n",
        "INDIC = [\"as\", \"bn\", \"gu\", \"hi\", \"kn\", \"ml\", \"mr\", \"or\", \"pa\", \"ta\", \"te\"]\n",
        "\n",
        "def split_sentences(paragraph, language):\n",
        "    if language == \"en\":\n",
        "        with MosesSentenceSplitter(language) as splitter:\n",
        "            return splitter([paragraph])\n",
        "    elif language in INDIC:\n",
        "        return sentence_tokenize.sentence_split(paragraph, lang=language)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "o5Ed0K3rrwDJ"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "```python\n",
        "split_sentences(\"\"\"COVID-19 is caused by infection with the severe acute respiratory\n",
        "syndrome coronavirus 2 (SARS-CoV-2) virus strain. The disease is mainly transmitted via the respiratory\n",
        "route when people inhale droplets and particles that infected people release as they breathe, talk, cough, sneeze, or sing. \"\"\", language='en')\n",
        "\n",
        "# >> ['COVID-19 is caused by infection with the severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2) virus strain.',\n",
        "#  'The disease is mainly transmitted via the respiratory route when people inhale droplets and particles that infected people release as they breathe, talk, cough, sneeze, or sing.']\n",
        "\n",
        "split_sentences(\"\"\"இத்தொற்றுநோய் உலகளாவிய சமூக மற்றும் பொருளாதார சீர்குலைவை ஏற்படுத்தியுள்ளது.இதனால் பெரும் பொருளாதார மந்தநிலைக்குப் பின்னர் உலகளவில் மிகப்பெரிய மந்தநிலை ஏற்பட்டுள்ளது. இது விளையாட்டு,மத, அரசியல் மற்றும் கலாச்சார நிகழ்வுகளை ஒத்திவைக்க அல்லது ரத்து செய்ய வழிவகுத்தது.\n",
        "அச்சம் காரணமாக முகக்கவசம், கிருமிநாசினி உள்ளிட்ட பொருட்களை அதிக நபர்கள் வாங்கியதால் விநியோகப் பற்றாக்குறை ஏற்பட்டது.\"\"\",\n",
        " language='ta')\n",
        "\n",
        "# >> ['இத்தொற்றுநோய் உலகளாவிய சமூக மற்றும் பொருளாதார சீர்குலைவை ஏற்படுத்தியுள்ளது.',\n",
        "#  'இதனால் பெரும் பொருளாதார மந்தநிலைக்குப் பின்னர் உலகளவில் மிகப்பெரிய மந்தநிலை ஏற்பட்டுள்ளது.',\n",
        "#  'இது விளையாட்டு,மத, அரசியல் மற்றும் கலாச்சார நிகழ்வுகளை ஒத்திவைக்க அல்லது ரத்து செய்ய வழிவகுத்தது.',\n",
        "#  'அச்சம் காரணமாக முகக்கவசம், கிருமிநாசினி உள்ளிட்ட பொருட்களை அதிக நபர்கள் வாங்கியதால் விநியோகப் பற்றாக்குறை ஏற்பட்டது.']\n",
        "```"
      ],
      "metadata": {
        "id": "0IziaFnrtcld"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sample_paragraph = \"Pongal, is also referred to as Thai Pongal, is a multi-day Hindu harvest festival celebrated by Tamils in India and Sri Lanka. It is observed at the start of the month Tai according to Tamil solar calendar, and this is typically about January 14.\"\n",
        "\n",
        "sentence_batch = split_sentences(sample_paragraph, language=\"en\")"
      ],
      "metadata": {
        "id": "K-KWFMH5tbzv"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# once you have the paragraph converted to batch of sentences, you can use the batch_translate endpoint which we saw previously\n",
        "\n",
        "# here we are translating the english sentences to hindi\n",
        "response = requests.post(\n",
        "    API_URL,\n",
        "    json={\n",
        "  \"text_lines\": sentence_batch,\n",
        "  \"source_language\": \"en\",\n",
        "  \"target_language\": \"hi\"\n",
        "},\n",
        ")\n",
        "\n",
        "output = json.loads(response.text)"
      ],
      "metadata": {
        "id": "VHnmSEqjtHT-"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(output[\"text_lines\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZthS_DAnuEe9",
        "outputId": "01a78ee7-f9bd-46db-f78c-2b115da133b8"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['पोंगल, जिसे थाई पोंगल के रूप में भी जाना जाता है, भारत और श्रीलंका में तमिलों द्वारा मनाया जाने वाला एक बहु-दिवसीय हिंदू फसल त्योहार है।', 'यह तमिल सौर कैलेंडर के अनुसार ताई महीने की शुरुआत में मनाया जाता है, और यह आमतौर पर 14 जनवरी को मनाया जाता है।']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"source sentence in english -----> Hindi Translation\")\n",
        "print()\n",
        "for src_sen, tgt_sen in zip(sentence_batch, output[\"text_lines\"]):\n",
        "    print(f\"{src_sen} -----> {tgt_sen}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YYeIFH-WuLrp",
        "outputId": "0fc8e0a5-675f-4f49-a6a3-51928d9c1119"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "source sentence in english -----> Hindi Translation\n",
            "\n",
            "Pongal, is also referred to as Thai Pongal, is a multi-day Hindu harvest festival celebrated by Tamils in India and Sri Lanka. -----> पोंगल, जिसे थाई पोंगल के रूप में भी जाना जाता है, भारत और श्रीलंका में तमिलों द्वारा मनाया जाने वाला एक बहु-दिवसीय हिंदू फसल त्योहार है।\n",
            "It is observed at the start of the month Tai according to Tamil solar calendar, and this is typically about January 14. -----> यह तमिल सौर कैलेंडर के अनुसार ताई महीने की शुरुआत में मनाया जाता है, और यह आमतौर पर 14 जनवरी को मनाया जाता है।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "9E25IJqcujYD"
      },
      "execution_count": 14,
      "outputs": []
    }
  ]
}