1.3_parsing_criminal_court_orders.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import gzip\n",
    "import json\n",
    "from glob import glob\n",
    "\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from pymystem3 import Mystem\n",
    "from many_stop_words import get_stop_words\n",
    "pd.set_option('display.max_colwidth', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "criminal_code = pd.read_csv('tmp/criminal_code.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.GzipFile('data/criminal_code/court_orders.json.gz', 'r') as f:\n",
    "    df = pd.DataFrame(list(map(lambda line: json.loads(line), f)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['title'] = df['title'].map(lambda x: x[0])\n",
    "df['body']= df['body'].map(lambda x: x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_table(table_body):\n",
    "    data = dict()\n",
    "    for line in BeautifulSoup(table_body, 'lxml').find_all('tr'):\n",
    "        data[line.find('td').get_text()] = line.find('th').get_text()\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Выкидываем записи с пустой таблицей метаданных:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df[df['table'].map(lambda x: len(x)==0)].index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Парсим таблицу метаданных:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['table'] = df['table'].map(lambda x: parse_table(x[0]))\n",
    "df = df.join(df['table'].apply(pd.Series)).drop('table', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Фильтруем только статьи с вынесением приговора:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "parts_pattern = re.compile(r'\\s?у\\s?с\\s?т\\s?а\\s?н\\s?о\\s?в\\s?и\\s?л\\s?:\\s?|\\s?п\\s?р\\s?и\\s?г\\s?о\\s?в\\s?о\\s?р\\s?и\\s?л\\s?:\\s?|\\s?п\\s?о\\s?с\\s?т\\s?а\\s?н\\s?о\\s?в\\s?и\\s?л\\s?:\\s?', flags=re.UNICODE | re.IGNORECASE)\n",
    "def split_parts(text):\n",
    "    parts = re.split(parts_pattern, text)\n",
    "    return tuple(map(str.strip, parts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['parts'] = df['body'].map(split_parts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df[df['parts'].map(len) != 3].index).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['intro'], df['case'], df['result'] = df['parts'].str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(columns=['parts'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Из решения суда выделяем номера статей кодекса:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = re.compile(r'(статьей|ст\\.|статьи)\\s*(\\d+\\.?\\d*)', flags=re.UNICODE | re.IGNORECASE)\n",
    "df['labels'] = df['result'].map(lambda text: list(map(lambda x: float(x[1]), re.findall(pattern, text))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df['labels'].map(len) != 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Оставляем только те статьи кодекса, которые мы знаем:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "only_known = lambda arr: list(filter(lambda x: x in criminal_code['article_number'].tolist(), arr))\n",
    "df['labels'] = df['labels'].map(only_known)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df['labels'].map(len) != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[:, 'labels'] = df['labels'].map(lambda x: ' '.join(list(map(str, x))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={\n",
    "    'Категория':'category',\n",
    "    'Дата':'date',\n",
    "    'Регион':'region',\n",
    "    'Суд':'court',\n",
    "    'Судья': 'judge',\n",
    "})\n",
    "df = df[['title', 'date', 'category', 'region', 'court', 'judge', 'body', 'intro', 'case', 'result', 'labels']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Количество судебных решений  25696\n"
     ]
    }
   ],
   "source": [
    "print('Количество судебных решений ', df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Mystem(entire_input=False)\n",
    "ru_stopwords= get_stop_words('ru')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemm_and_filter(text):\n",
    "    lemmas = m.lemmatize(text)\n",
    "    filter_cond = lambda word: word not in ru_stopwords and len(word) > 3\n",
    "    return ' '.join(list(filter(filter_cond, lemmas)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['lemmas'] = df['case'].map(lemm_and_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('tmp/criminal_code_court_orders.csv.gz', index=False, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (testing)",
   "language": "python",
   "name": "testing"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}