Morph_analyzer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Morph_analyzer",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1ZEthQ967ffOtn_KU7C0Yn6RdGXGYuG11",
      "authorship_tag": "ABX9TyMLdiue6nWeF/BNXwaUJmj0"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c4NE-2J99in4"
      },
      "source": [
        "**Задача:** Лемматизировать текст (привести к словарной форме) и приписать леммам частеречные теги. \n",
        "Для решения задачи вы можете использовать данные, которые упоминались в лекциях: например, словарь oDict, разметку OpenCorpora и др. Для решения задачи нельзя использовать существующие морфологические анализаторы (mystem, pymorphy и т.п.).\n",
        "\n",
        "**Ввод:** предложения вида \"токен1 токен2 ... токенN\" с расставленными знаками препинания, разделенные переносом строки. Из знаков препинания в предложениях могут содержаться только запятая, точка, вопросительный и восклицательный знаки.\n",
        "\n",
        "**Вывод:** для каждого предложения из входных данных вывод в виде \"токен1{лемма1=тег1} токен2{лемма2= тег2} ... токенN{леммаN=тегN}\" без исходных знаков препинания. Разделитель между токенами -- пробельный символ.\n",
        "\n",
        "Full description: https://stepik.org/lesson/37845/step/1?unit=18887"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JFeQOqqrtRyw"
      },
      "source": [
        "from bs4 import BeautifulSoup\n",
        "from tqdm import tqdm\n",
        "import requests\n",
        "import re"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vo1Lz2owHoYl"
      },
      "source": [
        "def read_txt(path):\n",
        "  with open(path, encoding=\"utf-8\") as file:\n",
        "    return [l.rstrip(\"\\n\") for l in file]"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UvruJw-iKX7c"
      },
      "source": [
        "def write_txt(path, data_):\n",
        "  with open(path, 'w') as output:\n",
        "    for row in data_:\n",
        "      output.write(row + '\\n')"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hmJfS9WwZXa"
      },
      "source": [
        "def morph(word):\n",
        "  link = 'https://morfologija.ru/словоформа/'   # ссылка на online словарь\n",
        "  request = requests.get(link+word)\n",
        "  soup = BeautifulSoup(request.text, 'lxml')\n",
        "  try:\n",
        "    lem = soup.td.get_text()\n",
        "    pos = soup.find('b').next_sibling\n",
        "    return lem,pos\n",
        "  except:\n",
        "    return word, ' существительное'      # тк по статистически сущ чаще встречаются (для слов отсутствующих в словаре)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uXS9seaREJML"
      },
      "source": [
        "def lemmatize(word_):\n",
        "  pos_tag = {' существительное': 'S',\n",
        "           ' инфинитив - ': 'V', \n",
        "           ' прилагательное': 'A', \n",
        "           ' кр. прилагательное': 'A',\n",
        "           ' предлог - ': 'PR', \n",
        "           ' союз - ': 'CONJ', \n",
        "           ' наречие - ': 'ADV',\n",
        "           ' междометие - ': 'ADV',\n",
        "           ' вводное слово - ': 'ADV',\n",
        "           ' частица - ': 'ADV',\n",
        "           ' местоимение': 'NI',\n",
        "           ' местоим. прил.': 'NI',\n",
        "           ' числительное': 'NI',\n",
        "           ' поряд. числительное': 'NI',\n",
        "           ' предикатив - ': 'NI',\n",
        "           ' ФРАЗ - ': 'NI'}\n",
        "  w1 = word_\n",
        "  w2 = morph(word_)[0]\n",
        "  w3 = pos_tag[morph(word_)[1]]\n",
        "  return '{}{{{}={}}}'.format(w1,w2,w3)\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ySbI32KEJOO"
      },
      "source": [
        "def preprocess(string):\n",
        "  string = re.sub(r'[^\\w\\s]','',string,flags=re.UNICODE)\n",
        "  string = re.sub(r'[\\d]','',string,flags=re.UNICODE) # digits\n",
        "  return string"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uv3Ut_7Hoa6"
      },
      "source": [
        "data = read_txt('/content/dataset_37845_1.txt')"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rYwMp-rbq14D",
        "outputId": "60202cf5-1cb1-48ff-e287-18dd9be5ad73"
      },
      "source": [
        "print(len(data))\n",
        "print(data[0])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "200\n",
            "Однако следует иметь в виду, что в интегральной компоновке применяется только отработанное высоконадежное насыщающее оборудование, поскольку по ремонтопригодности она заметно уступает и петлевой, и блочной компоновкам.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ghzPLidizVO",
        "outputId": "62790a9b-67f8-4523-f973-051cf37f3a07"
      },
      "source": [
        "processed_data = []\n",
        "tmp_dict = {}\n",
        "\n",
        "for sentence in tqdm(data):\n",
        "  tmp = ''\n",
        "  for w in preprocess(sentence).split():\n",
        "    if w in tmp_dict:\n",
        "      tmp = tmp + tmp_dict[w] + ' '\n",
        "\n",
        "    else:\n",
        "      lemma = lemmatize(w)\n",
        "      tmp = tmp + lemma + ' '\n",
        "      tmp_dict[w] = lemma\n",
        "  processed_data += [tmp[:-1]]"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/200 [00:00<?, ?it/s]\u001b[A\n",
            "  0%|          | 1/200 [00:36<2:02:08, 36.82s/it]\u001b[A\n",
            "  1%|          | 2/200 [01:05<1:53:55, 34.52s/it]\u001b[A\n",
            "  2%|▏         | 3/200 [01:35<1:48:37, 33.08s/it]\u001b[A\n",
            "  2%|▏         | 4/200 [01:51<1:31:14, 27.93s/it]\u001b[A\n",
            "  2%|▎         | 5/200 [02:21<1:32:58, 28.61s/it]\u001b[A\n",
            "  3%|▎         | 6/200 [02:48<1:30:11, 27.89s/it]\u001b[A\n",
            "  4%|▎         | 7/200 [03:22<1:36:22, 29.96s/it]\u001b[A\n",
            "  4%|▍         | 8/200 [03:41<1:24:46, 26.49s/it]\u001b[A\n",
            "  4%|▍         | 9/200 [04:10<1:26:39, 27.22s/it]\u001b[A\n",
            "  5%|▌         | 10/200 [04:26<1:15:37, 23.88s/it]\u001b[A\n",
            "  6%|▌         | 11/200 [04:57<1:21:56, 26.01s/it]\u001b[A\n",
            "  6%|▌         | 12/200 [05:14<1:13:26, 23.44s/it]\u001b[A\n",
            "  6%|▋         | 13/200 [05:29<1:05:16, 20.94s/it]\u001b[A\n",
            "  7%|▋         | 14/200 [05:40<55:05, 17.77s/it]  \u001b[A\n",
            "  8%|▊         | 15/200 [05:50<48:21, 15.68s/it]\u001b[A\n",
            "  8%|▊         | 16/200 [06:01<43:15, 14.11s/it]\u001b[A\n",
            "  8%|▊         | 17/200 [06:12<40:17, 13.21s/it]\u001b[A\n",
            "  9%|▉         | 18/200 [06:25<40:01, 13.20s/it]\u001b[A\n",
            " 10%|▉         | 19/200 [06:39<39:57, 13.25s/it]\u001b[A\n",
            " 10%|█         | 20/200 [06:56<43:37, 14.54s/it]\u001b[A\n",
            " 10%|█         | 21/200 [07:24<55:13, 18.51s/it]\u001b[A\n",
            " 11%|█         | 22/200 [07:37<50:12, 16.92s/it]\u001b[A\n",
            " 12%|█▏        | 23/200 [08:14<1:07:34, 22.91s/it]\u001b[A\n",
            " 12%|█▏        | 24/200 [08:26<57:57, 19.76s/it]  \u001b[A\n",
            " 12%|█▎        | 25/200 [08:42<54:01, 18.52s/it]\u001b[A\n",
            " 13%|█▎        | 26/200 [09:04<56:58, 19.65s/it]\u001b[A\n",
            " 14%|█▎        | 27/200 [09:30<1:01:37, 21.37s/it]\u001b[A\n",
            " 14%|█▍        | 28/200 [09:40<51:46, 18.06s/it]  \u001b[A\n",
            " 14%|█▍        | 29/200 [09:56<49:35, 17.40s/it]\u001b[A\n",
            " 15%|█▌        | 30/200 [10:12<48:17, 17.05s/it]\u001b[A\n",
            " 16%|█▌        | 31/200 [10:27<45:58, 16.32s/it]\u001b[A\n",
            " 16%|█▌        | 32/200 [10:51<52:16, 18.67s/it]\u001b[A\n",
            " 16%|█▋        | 33/200 [11:11<53:32, 19.23s/it]\u001b[A\n",
            " 17%|█▋        | 34/200 [11:30<52:42, 19.05s/it]\u001b[A\n",
            " 18%|█▊        | 35/200 [11:50<53:05, 19.30s/it]\u001b[A\n",
            " 18%|█▊        | 36/200 [12:10<53:06, 19.43s/it]\u001b[A\n",
            " 18%|█▊        | 37/200 [12:40<1:01:44, 22.73s/it]\u001b[A\n",
            " 19%|█▉        | 38/200 [13:05<1:02:47, 23.25s/it]\u001b[A\n",
            " 20%|█▉        | 39/200 [13:26<1:01:13, 22.82s/it]\u001b[A\n",
            " 20%|██        | 40/200 [13:45<57:19, 21.50s/it]  \u001b[A\n",
            " 20%|██        | 41/200 [14:00<51:50, 19.56s/it]\u001b[A\n",
            " 21%|██        | 42/200 [14:14<46:54, 17.81s/it]\u001b[A\n",
            " 22%|██▏       | 43/200 [14:31<46:10, 17.64s/it]\u001b[A\n",
            " 22%|██▏       | 44/200 [14:44<42:14, 16.25s/it]\u001b[A\n",
            " 22%|██▎       | 45/200 [15:00<41:50, 16.20s/it]\u001b[A\n",
            " 23%|██▎       | 46/200 [15:23<46:32, 18.13s/it]\u001b[A\n",
            " 24%|██▎       | 47/200 [15:42<47:21, 18.57s/it]\u001b[A\n",
            " 24%|██▍       | 48/200 [16:23<1:03:50, 25.20s/it]\u001b[A\n",
            " 24%|██▍       | 49/200 [16:41<57:50, 22.99s/it]  \u001b[A\n",
            " 25%|██▌       | 50/200 [16:59<53:43, 21.49s/it]\u001b[A\n",
            " 26%|██▌       | 51/200 [17:14<49:03, 19.76s/it]\u001b[A\n",
            " 26%|██▌       | 52/200 [17:34<48:25, 19.63s/it]\u001b[A\n",
            " 26%|██▋       | 53/200 [17:50<45:37, 18.62s/it]\u001b[A\n",
            " 27%|██▋       | 54/200 [17:59<38:29, 15.82s/it]\u001b[A\n",
            " 28%|██▊       | 55/200 [18:09<33:30, 13.87s/it]\u001b[A\n",
            " 28%|██▊       | 56/200 [18:28<37:28, 15.61s/it]\u001b[A\n",
            " 28%|██▊       | 57/200 [18:35<30:54, 12.97s/it]\u001b[A\n",
            " 29%|██▉       | 58/200 [18:48<30:24, 12.85s/it]\u001b[A\n",
            " 30%|██▉       | 59/200 [19:04<33:01, 14.06s/it]\u001b[A\n",
            " 30%|███       | 60/200 [19:24<36:55, 15.83s/it]\u001b[A\n",
            " 30%|███       | 61/200 [19:50<43:25, 18.75s/it]\u001b[A\n",
            " 31%|███       | 62/200 [20:02<38:48, 16.87s/it]\u001b[A\n",
            " 32%|███▏      | 63/200 [20:15<35:14, 15.43s/it]\u001b[A\n",
            " 32%|███▏      | 64/200 [20:42<42:49, 18.90s/it]\u001b[A\n",
            " 32%|███▎      | 65/200 [20:53<37:33, 16.69s/it]\u001b[A\n",
            " 33%|███▎      | 66/200 [21:09<36:56, 16.54s/it]\u001b[A\n",
            " 34%|███▎      | 67/200 [21:32<40:43, 18.37s/it]\u001b[A\n",
            " 34%|███▍      | 68/200 [21:45<37:05, 16.86s/it]\u001b[A\n",
            " 34%|███▍      | 69/200 [22:18<47:27, 21.74s/it]\u001b[A\n",
            " 35%|███▌      | 70/200 [23:08<1:05:02, 30.02s/it]\u001b[A\n",
            " 36%|███▌      | 71/200 [23:33<1:01:10, 28.45s/it]\u001b[A\n",
            " 36%|███▌      | 72/200 [23:39<46:37, 21.86s/it]  \u001b[A\n",
            " 36%|███▋      | 73/200 [24:15<55:02, 26.01s/it]\u001b[A\n",
            " 37%|███▋      | 74/200 [24:31<48:49, 23.25s/it]\u001b[A\n",
            " 38%|███▊      | 75/200 [24:43<40:52, 19.62s/it]\u001b[A\n",
            " 38%|███▊      | 76/200 [24:52<34:25, 16.66s/it]\u001b[A\n",
            " 38%|███▊      | 77/200 [25:29<46:40, 22.77s/it]\u001b[A\n",
            " 39%|███▉      | 78/200 [25:52<46:12, 22.72s/it]\u001b[A\n",
            " 40%|███▉      | 79/200 [26:16<46:23, 23.00s/it]\u001b[A\n",
            " 40%|████      | 80/200 [26:26<38:21, 19.18s/it]\u001b[A\n",
            " 40%|████      | 81/200 [27:13<54:41, 27.58s/it]\u001b[A\n",
            " 41%|████      | 82/200 [27:39<53:03, 26.98s/it]\u001b[A\n",
            " 42%|████▏     | 83/200 [27:49<42:35, 21.84s/it]\u001b[A\n",
            " 42%|████▏     | 84/200 [28:22<49:15, 25.47s/it]\u001b[A\n",
            " 42%|████▎     | 85/200 [28:44<46:45, 24.40s/it]\u001b[A\n",
            " 43%|████▎     | 86/200 [28:52<36:57, 19.45s/it]\u001b[A\n",
            " 44%|████▎     | 87/200 [29:02<30:56, 16.43s/it]\u001b[A\n",
            " 44%|████▍     | 88/200 [29:13<27:44, 14.86s/it]\u001b[A\n",
            " 44%|████▍     | 89/200 [29:25<25:42, 13.89s/it]\u001b[A\n",
            " 45%|████▌     | 90/200 [29:41<26:46, 14.60s/it]\u001b[A\n",
            " 46%|████▌     | 91/200 [29:56<26:46, 14.74s/it]\u001b[A\n",
            " 46%|████▌     | 92/200 [30:10<26:06, 14.51s/it]\u001b[A\n",
            " 46%|████▋     | 93/200 [30:35<31:23, 17.60s/it]\u001b[A\n",
            " 47%|████▋     | 94/200 [31:15<43:08, 24.42s/it]\u001b[A\n",
            " 48%|████▊     | 95/200 [31:25<35:03, 20.03s/it]\u001b[A\n",
            " 48%|████▊     | 96/200 [31:37<30:49, 17.79s/it]\u001b[A\n",
            " 48%|████▊     | 97/200 [31:50<27:53, 16.25s/it]\u001b[A\n",
            " 49%|████▉     | 98/200 [32:08<28:25, 16.72s/it]\u001b[A\n",
            " 50%|████▉     | 99/200 [32:21<26:22, 15.67s/it]\u001b[A\n",
            " 50%|█████     | 100/200 [32:48<31:42, 19.03s/it]\u001b[A\n",
            " 50%|█████     | 101/200 [32:59<27:31, 16.68s/it]\u001b[A\n",
            " 51%|█████     | 102/200 [33:11<24:42, 15.13s/it]\u001b[A\n",
            " 52%|█████▏    | 103/200 [33:24<23:40, 14.64s/it]\u001b[A\n",
            " 52%|█████▏    | 104/200 [33:50<28:51, 18.04s/it]\u001b[A\n",
            " 52%|█████▎    | 105/200 [34:02<25:29, 16.10s/it]\u001b[A\n",
            " 53%|█████▎    | 106/200 [34:16<24:20, 15.54s/it]\u001b[A\n",
            " 54%|█████▎    | 107/200 [34:22<19:53, 12.83s/it]\u001b[A\n",
            " 54%|█████▍    | 108/200 [34:36<20:13, 13.19s/it]\u001b[A\n",
            " 55%|█████▍    | 109/200 [34:50<20:19, 13.40s/it]\u001b[A\n",
            " 55%|█████▌    | 110/200 [34:59<18:12, 12.14s/it]\u001b[A\n",
            " 56%|█████▌    | 111/200 [35:17<20:13, 13.63s/it]\u001b[A\n",
            " 56%|█████▌    | 112/200 [35:35<22:13, 15.15s/it]\u001b[A\n",
            " 56%|█████▋    | 113/200 [35:52<22:32, 15.54s/it]\u001b[A\n",
            " 57%|█████▋    | 114/200 [36:03<20:22, 14.22s/it]\u001b[A\n",
            " 57%|█████▊    | 115/200 [36:19<21:08, 14.92s/it]\u001b[A\n",
            " 58%|█████▊    | 116/200 [36:39<22:39, 16.19s/it]\u001b[A\n",
            " 58%|█████▊    | 117/200 [36:48<19:33, 14.14s/it]\u001b[A\n",
            " 59%|█████▉    | 118/200 [37:03<19:41, 14.40s/it]\u001b[A\n",
            " 60%|█████▉    | 119/200 [37:20<20:42, 15.34s/it]\u001b[A\n",
            " 60%|██████    | 120/200 [37:30<18:01, 13.52s/it]\u001b[A\n",
            " 60%|██████    | 121/200 [37:41<16:59, 12.90s/it]\u001b[A\n",
            " 61%|██████    | 122/200 [37:57<18:04, 13.90s/it]\u001b[A\n",
            " 62%|██████▏   | 123/200 [38:08<16:33, 12.90s/it]\u001b[A\n",
            " 62%|██████▏   | 124/200 [38:22<16:36, 13.12s/it]\u001b[A\n",
            " 62%|██████▎   | 125/200 [38:30<14:34, 11.66s/it]\u001b[A\n",
            " 63%|██████▎   | 126/200 [39:09<24:24, 19.79s/it]\u001b[A\n",
            " 64%|██████▎   | 127/200 [39:23<22:11, 18.23s/it]\u001b[A\n",
            " 64%|██████▍   | 128/200 [39:39<20:51, 17.38s/it]\u001b[A\n",
            " 64%|██████▍   | 129/200 [39:58<21:15, 17.96s/it]\u001b[A\n",
            " 65%|██████▌   | 130/200 [40:14<20:10, 17.30s/it]\u001b[A\n",
            " 66%|██████▌   | 131/200 [40:27<18:35, 16.17s/it]\u001b[A\n",
            " 66%|██████▌   | 132/200 [40:53<21:36, 19.07s/it]\u001b[A\n",
            " 66%|██████▋   | 133/200 [41:21<24:17, 21.75s/it]\u001b[A\n",
            " 67%|██████▋   | 134/200 [41:51<26:41, 24.27s/it]\u001b[A\n",
            " 68%|██████▊   | 135/200 [41:55<19:37, 18.12s/it]\u001b[A\n",
            " 68%|██████▊   | 136/200 [42:02<15:42, 14.73s/it]\u001b[A\n",
            " 68%|██████▊   | 137/200 [42:16<15:24, 14.68s/it]\u001b[A\n",
            " 69%|██████▉   | 138/200 [42:33<15:55, 15.41s/it]\u001b[A\n",
            " 70%|██████▉   | 139/200 [42:54<17:10, 16.90s/it]\u001b[A\n",
            " 70%|███████   | 140/200 [43:08<16:09, 16.16s/it]\u001b[A\n",
            " 70%|███████   | 141/200 [43:37<19:37, 19.95s/it]\u001b[A\n",
            " 71%|███████   | 142/200 [43:49<16:54, 17.49s/it]\u001b[A\n",
            " 72%|███████▏  | 143/200 [44:03<15:43, 16.55s/it]\u001b[A\n",
            " 72%|███████▏  | 144/200 [44:19<15:13, 16.31s/it]\u001b[A\n",
            " 72%|███████▎  | 145/200 [44:31<13:40, 14.92s/it]\u001b[A\n",
            " 73%|███████▎  | 146/200 [44:47<13:46, 15.30s/it]\u001b[A\n",
            " 74%|███████▎  | 147/200 [45:03<13:45, 15.57s/it]\u001b[A\n",
            " 74%|███████▍  | 148/200 [45:16<12:53, 14.88s/it]\u001b[A\n",
            " 74%|███████▍  | 149/200 [45:29<12:09, 14.30s/it]\u001b[A\n",
            " 75%|███████▌  | 150/200 [45:41<11:12, 13.45s/it]\u001b[A\n",
            " 76%|███████▌  | 151/200 [45:59<12:15, 15.01s/it]\u001b[A\n",
            " 76%|███████▌  | 152/200 [46:12<11:20, 14.17s/it]\u001b[A\n",
            " 76%|███████▋  | 153/200 [46:35<13:19, 17.02s/it]\u001b[A\n",
            " 77%|███████▋  | 154/200 [46:55<13:38, 17.80s/it]\u001b[A\n",
            " 78%|███████▊  | 155/200 [47:00<10:33, 14.07s/it]\u001b[A\n",
            " 78%|███████▊  | 156/200 [47:30<13:52, 18.93s/it]\u001b[A\n",
            " 78%|███████▊  | 157/200 [47:40<11:30, 16.05s/it]\u001b[A\n",
            " 79%|███████▉  | 158/200 [47:45<08:52, 12.68s/it]\u001b[A\n",
            " 80%|███████▉  | 159/200 [47:49<07:01, 10.27s/it]\u001b[A\n",
            " 80%|████████  | 160/200 [48:03<07:31, 11.30s/it]\u001b[A\n",
            " 80%|████████  | 161/200 [48:23<08:57, 13.78s/it]\u001b[A\n",
            " 81%|████████  | 162/200 [48:30<07:32, 11.90s/it]\u001b[A\n",
            " 82%|████████▏ | 163/200 [48:41<07:08, 11.59s/it]\u001b[A\n",
            " 82%|████████▏ | 164/200 [48:51<06:43, 11.20s/it]\u001b[A\n",
            " 82%|████████▎ | 165/200 [49:10<07:50, 13.44s/it]\u001b[A\n",
            " 83%|████████▎ | 166/200 [49:26<08:01, 14.16s/it]\u001b[A\n",
            " 84%|████████▎ | 167/200 [49:51<09:41, 17.63s/it]\u001b[A\n",
            " 84%|████████▍ | 168/200 [50:01<08:07, 15.24s/it]\u001b[A\n",
            " 84%|████████▍ | 169/200 [50:16<07:47, 15.07s/it]\u001b[A\n",
            " 85%|████████▌ | 170/200 [50:28<07:07, 14.24s/it]\u001b[A\n",
            " 86%|████████▌ | 171/200 [50:37<06:04, 12.58s/it]\u001b[A\n",
            " 86%|████████▌ | 172/200 [50:57<06:58, 14.95s/it]\u001b[A\n",
            " 86%|████████▋ | 173/200 [51:15<07:04, 15.72s/it]\u001b[A\n",
            " 87%|████████▋ | 174/200 [51:26<06:09, 14.21s/it]\u001b[A\n",
            " 88%|████████▊ | 175/200 [51:44<06:24, 15.37s/it]\u001b[A\n",
            " 88%|████████▊ | 176/200 [51:58<06:01, 15.08s/it]\u001b[A\n",
            " 88%|████████▊ | 177/200 [52:10<05:26, 14.20s/it]\u001b[A\n",
            " 89%|████████▉ | 178/200 [52:21<04:47, 13.07s/it]\u001b[A\n",
            " 90%|████████▉ | 179/200 [52:35<04:45, 13.60s/it]\u001b[A\n",
            " 90%|█████████ | 180/200 [52:47<04:22, 13.14s/it]\u001b[A\n",
            " 90%|█████████ | 181/200 [53:04<04:28, 14.11s/it]\u001b[A\n",
            " 91%|█████████ | 182/200 [53:22<04:33, 15.18s/it]\u001b[A\n",
            " 92%|█████████▏| 183/200 [53:34<04:05, 14.43s/it]\u001b[A\n",
            " 92%|█████████▏| 184/200 [53:43<03:25, 12.84s/it]\u001b[A\n",
            " 92%|█████████▎| 185/200 [53:56<03:09, 12.66s/it]\u001b[A\n",
            " 93%|█████████▎| 186/200 [54:05<02:44, 11.78s/it]\u001b[A\n",
            " 94%|█████████▎| 187/200 [54:27<03:12, 14.78s/it]\u001b[A\n",
            " 94%|█████████▍| 188/200 [54:40<02:50, 14.24s/it]\u001b[A\n",
            " 94%|█████████▍| 189/200 [54:45<02:04, 11.35s/it]\u001b[A\n",
            " 95%|█████████▌| 190/200 [54:57<01:56, 11.62s/it]\u001b[A\n",
            " 96%|█████████▌| 191/200 [55:16<02:04, 13.81s/it]\u001b[A\n",
            " 96%|█████████▌| 192/200 [55:31<01:53, 14.14s/it]\u001b[A\n",
            " 96%|█████████▋| 193/200 [55:53<01:56, 16.58s/it]\u001b[A\n",
            " 97%|█████████▋| 194/200 [56:07<01:34, 15.68s/it]\u001b[A\n",
            " 98%|█████████▊| 195/200 [56:15<01:07, 13.54s/it]\u001b[A\n",
            " 98%|█████████▊| 196/200 [56:30<00:55, 13.87s/it]\u001b[A\n",
            " 98%|█████████▊| 197/200 [56:45<00:42, 14.27s/it]\u001b[A\n",
            " 99%|█████████▉| 198/200 [57:09<00:34, 17.19s/it]\u001b[A\n",
            "100%|█████████▉| 199/200 [57:18<00:14, 14.69s/it]\u001b[A\n",
            "100%|██████████| 200/200 [57:38<00:00, 17.29s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VY6G44JLoZ8O",
        "outputId": "7f6aecf5-d1ee-47de-98bf-4d13991fd29e"
      },
      "source": [
        "len(processed_data)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "200"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m2DJAUoIz1Gp"
      },
      "source": [
        "write_txt('/content/lemma_res.txt', processed_data)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "id": "ZlwXThR_40Aq",
        "outputId": "fa4379d4-53f1-4054-d9a4-4572d92a3308"
      },
      "source": [
        "processed_data[0]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Однако{однако=ADV} следует{следовать=V} иметь{иметь=V} в{в=PR} виду{вид=S} что{что=CONJ} в{в=PR} интегральной{интегральный=A} компоновке{компоновка=S} применяется{применяться=V} только{только=CONJ} отработанное{отработать=V} высоконадежное{высоконадежный=A} насыщающее{насыщать=V} оборудование{оборудование=S} поскольку{поскольку=CONJ} по{по=PR} ремонтопригодности{ремонтопригодность=S} она{он=NI} заметно{заметно=ADV} уступает{уступать=V} и{и=CONJ} петлевой{петлевой=A} и{и=CONJ} блочной{блочный=A} компоновкам{компоновка=S}'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    }
  ]
}