evaluation_srl.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Semantic Role Labeling metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import f1_score, confusion_matrix, recall_score, precision_score, roc_auc_score, accuracy_score\n",
    "import seaborn as sbn\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "arg_pred = np.load(\"./predictions_srl_det/argument_eval.npy\")\n",
    "arg_real = np.load(\"./predictions_srl_det/arg_real_eval.npy\")\n",
    "mask = np.load(\"./predictions_srl_det/f_mask_eval.npy\")\n",
    "detection = np.load(\"./predictions_srl_det/detection_eval.npy\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 583  \n",
      "FN: 122 \n",
      "FP_v: 122\n",
      "F_det: 61 \n",
      "precision= 0.761097 \n",
      "recall= 0.826950\n",
      "f1-score = 0.792658\n",
      "accuracy = 0.656532\n",
      "[[  0  61]\n",
      " [122 583]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fb837379860>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAJDCAYAAADKCVQfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFJZJREFUeJzt3V2spVdZB/D/Qz8kAYTECuJM0SIFKWoCaYoJJmBAnXLRXmi0JUYhDeOFNRo/Yo0GtN6IJpiY1I+T2KAktlYuzESrJakloFJsE7ShbaqTmtApJFWoXAhayn68mFM8DDNnn7Znn/Wu098v2cl53/2etdc0ncyT/3rW2tXdAQBYqueNngAAwG4UKwDAoilWAIBFU6wAAIumWAEAFk2xAgAsmmIFANg3VXVzVT1WVZ86x/tVVb9XVSer6r6qesO6MRUrAMB++kCSY7u8f2WSS7dfx5P8wboBFSsAwL7p7o8m+fwuj1yd5E/7tLuTvKSqXr7bmIoVAOAgHUnyyI7rU9v3zun8jU4nyfkXHnGePwzwpc98bPQU4DnrgoteWQf5eV/+z4cP7N/aC7/5O34qp5dvnrLV3Vub/MyNFysAwOGxXZg8m+Lk0SQX77g+un3vnCwDAQAH6USSn9jeFfS9Sb7Q3Z/d7RckKwAwu9VXRs/gq6rqliRvSXJRVZ1K8t4kFyRJd/9hktuTvD3JySRfTPKudWMqVgCAfdPd1655v5P89NMZU7ECALPr1egZbJSeFQBg0SQrADC7lWQFAGAYyQoATK71rAAAjCNZAYDZ6VkBABhHsgIAs9OzAgAwjmIFAFg0y0AAMLsFfZHhJkhWAIBFk6wAwOw02AIAjCNZAYDZORQOAGAcyQoATM4XGQIADCRZAYDZ6VkBABhHsgIAs9OzAgAwjmQFAGbnu4EAAMaRrADA7PSsAACMo1gBABbNMhAAzM6hcAAA40hWAGB2GmwBAMaRrADA7PSsAACMI1kBgMl1O24fAGAYyQoAzM5uIACAcSQrADA7u4EAAMaRrADA7PSsAACMI1kBgNmtnLMCADCMYgUAWDTLQAAwOw22AADjSFYAYHYOhQMAGEeyAgCz07MCADCOZAUAZqdnBQBgHMkKAMxOsgIAMI5kBQAm1+2LDAEAhpGsAMDs9KwAAIwjWQGA2TnBFgBgHMUKALBoloEAYHYabAEAxpGsAMDsNNgCAIwjWQGA2elZAQAYR7ICALPTswIAMI5kBQBmp2cFAGAcyQoAzE6yAgAwjmQFAGZnNxAAwDiSFQCYnZ4VAIBxFCsAwKJZBgKA2WmwBQAYR7ICALPTYAsAMI5kBQBmp2cFAGAcyQoAzE7PCgDAOJIVAJidZAUAYBzJCgDMrnv0DDZKsgIALJpkBQBmp2cFAGAcyQoAzE6yAgAwjmQFAGbnu4EAAMZRrAAAi2YZCABmp8EWAGAcxQoAzK774F5rVNWxqnqoqk5W1Q1nef8VVXVXVX2yqu6rqrevG1OxAgDsi6o6L8lNSa5MclmSa6vqsjMe+7Ukt3X365Nck+T3142rZwUAZrecnpUrkpzs7oeTpKpuTXJ1kgd2PNNJvnH75xcn+cy6QRUrAMB+OZLkkR3Xp5K88Yxnfj3Jh6vqZ5K8IMnb1g1qGQgAZrdaHdirqo5X1b07Xsef5myvTfKB7j6a5O1JPlhVu9YjkhUAYM+6eyvJ1jnefjTJxTuuj27f2+m6JMe2x/p4VT0/yUVJHjvXZ0pWAGB2vTq41+7uSXJpVV1SVRfmdAPtiTOe+XSStyZJVb02yfOT/MdugypWAIB90d1PJrk+yR1JHszpXT/3V9WNVXXV9mO/kOTdVfUvSW5J8s7u3fdEWwYCgMn1av35Jwelu29PcvsZ996z4+cHkrzp6YwpWQEAFk2yAgCzW845KxshWQEAFk2yAgCzW79LZ2qSFQBg0RQrAMCiWQYCgNktaOvyJkhWAIBFk6wAwOxsXQYAGEeyAgCzk6wAAIwjWQGA2e3+pcXTk6wAAIsmWQGA2elZAQAYR7ICALNzgi0AwDiSFQCYXR/unpW1xUpVfWeSq5Mc2b71aJIT3f3gJicGAJCsWQaqql9OcmuSSvJP269KcktV3bD56QEAa6364F4DrEtWrkvyuu7+8s6bVfX+JPcn+a1NTQwAIFnfYLtK8q1nuf/y7ffOqqqOV9W9VXXvavXfz2Z+AMBz3Lpk5eeS3FlV/5bkke17r0jyqiTXn+uXunsryVaSnH/hkcO9nwoAButDfijcrsVKd/9tVb06yRX52gbbe7r7K5ueHADA2t1A3b1KcvcBzAUAeCYcCgcAMI5D4QBgdof8UDjJCgCwaJIVAJidnhUAgHEkKwAwu0N+zopkBQBYNMkKAMxOzwoAwDiSFQCYnXNWAADGkawAwOz0rAAAjKNYAQAWzTIQAEyuHQoHADCOZAUAZqfBFgBgHMkKAMxOsgIAMI5kBQBm57h9AIBxJCsAMDs9KwAA40hWAGByLVkBABhHsgIAs5OsAACMI1kBgNn51mUAgHEUKwDAolkGAoDZabAFABhHsgIAs5OsAACMI1kBgMl1S1YAAIaRrADA7PSsAACMI1kBgNlJVgAAxpGsAMDkWrICADCOZAUAZidZAQAYR7ICALNbjZ7AZklWAIBFU6wAAItmGQgAJmfrMgDAQJIVAJidZAUAYBzJCgDMztZlAIBxJCsAMDm7gQAABpKsAMDs9KwAAIwjWQGAyelZAQAYSLICALPTswIAMI5kBQAm15IVAIBxFCsAwKJZBgKA2VkGAgAYR7ICAJPTYAsAMJBkBQBmJ1kBABhHsgIAk9OzAgAwkGQFACYnWQEA2KOqOlZVD1XVyaq64RzP/GhVPVBV91fVn60bU7ICAJNbSrJSVecluSnJDyQ5leSeqjrR3Q/seObSJL+S5E3d/XhVvXTduJIVAGC/XJHkZHc/3N1PJLk1ydVnPPPuJDd19+NJ0t2PrRtUsQIAs+s6uNfujiR5ZMf1qe17O706yaur6h+q6u6qOrZuUMtAAMCeVdXxJMd33Nrq7q2nMcT5SS5N8pYkR5N8tKq+u7v/a7dfAAAmdpA9K9uFybmKk0eTXLzj+uj2vZ1OJflEd385yb9X1b/mdPFyz7k+0zIQALBf7klyaVVdUlUXJrkmyYkznvnLnE5VUlUX5fSy0MO7DapYAQD2RXc/meT6JHckeTDJbd19f1XdWFVXbT92R5LPVdUDSe5K8kvd/bndxq3u3uS8c/6FRzb7AcBZfekzHxs9BXjOuuCiV67tRN1Pn/2+7z+wf2tf/vd3HeifLZGsAAALp8EWACa3lEPhNkWyAgAsmmQFACbX6w9rm5pkBQBYNMkKAExOzwoAwECSFQCYXK/0rAAADCNZAYDJbfgw+uEkKwDAoklWAGByelYAAAaSrADA5CQrAAADKVYAgEWzDAQAk7N1GQBgIMkKAExOgy0AwECSFQCYXLdkBQBgGMkKAEyuV6NnsFmSFQBg0SQrADC5lZ4VAIBxJCsAMDm7gQAABpKsAMDknGALADCQZAUAJudblwEABlKsAACLZhkIACanwRYAYCDJCgBMznH7AAADSVYAYHKO2wcAGEiyAgCTcygcAMBAkhUAmJzdQAAAA0lWAGBydgMBAAwkWQGAydkNBAAwkGQFACZnNxAAwEAbT1be/NLXbfojgLN48s4Pjp4CPGdd8GPvPdDPsxsIAGAgxQoAsGgabAFgchpsAQAGkqwAwOQO+ZlwkhUAYNkkKwAwOT0rAAADSVYAYHIOhQMAGEiyAgCTW42ewIZJVgCARZOsAMDkOnpWAACGkawAwORWh/wIW8kKALBokhUAmNxKzwoAwDiKFQBg0SwDAcDkbF0GABhIsgIAk3PcPgDAQJIVAJicnhUAgIEkKwAwOT0rAAADSVYAYHKSFQCAgSQrADA5u4EAAAaSrADA5FaHO1iRrAAAyyZZAYDJrfSsAACMo1gBABbNMhAATK5HT2DDJCsAwKJJVgBgco7bBwAYSLICAJNbla3LAADDSFYAYHJ2AwEADCRZAYDJ2Q0EADCQZAUAJrc63JuBJCsAwLJJVgBgcqsc7mhFsgIA7JuqOlZVD1XVyaq6YZfnfriquqouXzemYgUAJtcH+NpNVZ2X5KYkVya5LMm1VXXZWZ57UZKfTfKJvfz5FCsAwH65IsnJ7n64u59IcmuSq8/y3G8meV+S/9nLoIoVAGC/HEnyyI7rU9v3vqqq3pDk4u7+670OqsEWACZ3kFuXq+p4kuM7bm1199Yef/d5Sd6f5J1P5zMVKwDAnm0XJucqTh5NcvGO66Pb957yoiTfleQjdfqbor8lyYmquqq77z3XZypWAGByCzpu/54kl1bVJTldpFyT5B1PvdndX0hy0VPXVfWRJL+4W6GS6FkBAPZJdz+Z5PokdyR5MMlt3X1/Vd1YVVc903ElKwAwuXVbig9Sd9+e5PYz7r3nHM++ZS9jSlYAgEWTrADA5HyRIQDAQJIVAJjcgnYDbYRkBQBYNMkKAExOsgIAMJBkBQAm13YDAQCMI1kBgMnpWQEAGEixAgAsmmUgAJicZSAAgIEkKwAwuR49gQ2TrAAAiyZZAYDJrRwKBwAwjmQFACZnNxAAwECSFQCYnGQFAGAgyQoATM45KwAAA0lWAGByzlkBABhIsgIAk7MbCABgIMUKALBoloEAYHK2LgMADCRZAYDJrQ55tiJZAQAWTbICAJOzdRkAYCDJCgBM7nB3rEhWAICFk6wAwOT0rAAADCRZAYDJrWr0DDZLsgIALJpkBQAm5wRbAICBJCsAMLnDnatIVgCAhVOsAACLZhkIACbnUDgAgIEkKwAwOVuXAQAGkqwAwOQOd64iWQEAFk6yAgCTsxsIAGAgyQoATM5uIACAgSQrADC5w52rSFYAgIWTrADA5OwGAgAYSLICAJPrQ961IlkBABZNsQIALJplIACYnAZbAICBJCsAMDnH7QMADCRZAYDJHe5cRbICACycZAUAJqdnBQBgIMkKAEzOOSsAAANJVgBgcr7IEABgoGdcrFTVu/ZzIgDAM7M6wNcIzyZZ+Y19mwUAwDns2rNSVfed660kL9vl944nOZ4kr3nJa3PkBUef8QQBgN0d9p6VdQ22L0vyQ0keP+N+JfnHc/1Sd28l2UqStx79wcP9XxAA2Kh1xcpfJXlhd//zmW9U1Uc2MiMAgB12LVa6+7pd3nvH/k8HAHi6HAoHADCQQ+EAYHKrPtztoZIVAGDRJCsAMLnDnatIVgCAhZOsAMDkVoc8W5GsAACLJlkBgMkd9uP2JSsAwKJJVgBgck6wBQAYSLICAJOzGwgAYCDJCgBMzm4gAICBFCsAwKJZBgKAydm6DAAwkGQFACbXrcEWAGBPqupYVT1UVSer6oazvP/zVfVAVd1XVXdW1betG1OxAgCTW6UP7LWbqjovyU1JrkxyWZJrq+qyMx77ZJLLu/t7knwoyW+v+/MpVgCA/XJFkpPd/XB3P5Hk1iRX73ygu+/q7i9uX96d5Oi6QfWsAMDkFrQb6EiSR3Zcn0ryxl2evy7J36wbVLECAOxZVR1PcnzHra3u3noG4/x4ksuTvHnds4oVAJjcQR63v12YnKs4eTTJxTuuj27f+xpV9bYkv5rkzd39v+s+U88KALBf7klyaVVdUlUXJrkmyYmdD1TV65P8UZKruvuxvQwqWQGAya3bpXNQuvvJqro+yR1Jzktyc3ffX1U3Jrm3u08k+Z0kL0zyF1WVJJ/u7qt2G1exAgDsm+6+PcntZ9x7z46f3/Z0x1SsAMDknGALADCQZAUAJregc1Y2QrICACyaZAUAJneQ56yMIFkBABZNsQIALJplIACY3FIOhdsUyQoAsGiSFQCYnEPhAAAGkqwAwOT0rAAADCRZAYDJORQOAGAgyQoATG5lNxAAwDiSFQCY3OHOVSQrAMDCSVYAYHLOWQEAGEiyAgCTk6wAAAykWAEAFs0yEABMrh0KBwAwjmQFACanwRYAYCDJCgBMriUrAADjSFYAYHJ2AwEADCRZAYDJ2Q0EADCQZAUAJqdnBQBgIMkKAExOzwoAwECSFQCYnBNsAQAGUqwAAItmGQgAJreydRkAYBzJCgBMToMtAMBAkhUAmJyeFQCAgSQrADA5PSsAAANJVgBgcnpWAAAGkqwAwOT0rAAADCRZAYDJ6VkBABhIsgIAk9OzAgAwkGIFAFg0y0AAMLnu1egpbJRkBQBYNMkKAExupcEWAGAcyQoATK4dCgcAMI5kBQAmp2cFAGAgyQoATE7PCgDAQJIVAJjcSrICADCOZAUAJtd2AwEAjCNZAYDJ2Q0EADCQYgUAWDTLQAAwOcftAwAMJFkBgMlpsAUAGEiyAgCTc9w+AMBAkhUAmJyeFQCAgSQrADA556wAAAwkWQGAyelZAQAYSLICAJNzzgoAwECSFQCYXNsNBAAwjmIFAFg0y0AAMDkNtgAAA0lWAGByDoUDABhIsgIAk7N1GQBgIMkKAExOzwoAwECKFQCYXHcf2GudqjpWVQ9V1cmquuEs739DVf359vufqKpvXzemYgUA2BdVdV6Sm5JcmeSyJNdW1WVnPHZdkse7+1VJfjfJ+9aNq1gBgMn1Ab7WuCLJye5+uLufSHJrkqvPeObqJH+y/fOHkry1qmq3QRUrAMB+OZLkkR3Xp7bvnfWZ7n4yyReSfNNug258N9Cdpz68a7XEslXV8e7eGj0PeK7xd4+n48knHj2wf2ur6niS4ztubW36/1XJCuscX/8IsAH+7rFI3b3V3ZfveO0sVB5NcvGO66Pb93K2Z6rq/CQvTvK53T5TsQIA7Jd7klxaVZdU1YVJrkly4oxnTiT5ye2ffyTJ3/WabUYOhQMA9kV3P1lV1ye5I8l5SW7u7vur6sYk93b3iSR/nOSDVXUyyedzuqDZVR32U+94dqybwxj+7sH/U6wAAIumZwUAWDTFCme17rhkYDOq6uaqeqyqPjV6LrAUihW+zh6PSwY24wNJjo2eBCyJYoWz2ctxycAGdPdHc3qHBLBNscLZ7OW4ZAA4EIoVAGDRFCuczV6OSwaAA6FY4Wz2clwyABwIxQpfZ/sru586LvnBJLd19/1jZwXPDVV1S5KPJ3lNVZ2qqutGzwlGc4ItALBokhUAYNEUKwDAoilWAIBFU6wAAIumWAEAFk2xAgAsmmIFAFg0xQoAsGj/B7uVH/SiE7rbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "##### just argument detection\n",
    "\n",
    "a = np.exp(detection)\n",
    "probs = a / np.expand_dims(np.sum(a, axis=2),axis=-1)\n",
    "is_arg = (probs[:,:,1] > 0.5)\n",
    "fin_mask = (arg_real> 0) | is_arg\n",
    "\n",
    "\n",
    "y_pre, y_true  = (arg_real>0)[fin_mask].flatten(), is_arg[fin_mask].flatten()\n",
    "\n",
    "TP,FN , F_det, FP_v = 0,0,0,0\n",
    "for i,j in zip(y_pre, y_true):\n",
    "    if i==j and i!= 0:\n",
    "        TP += 1\n",
    "    if j!=0 and i==0:\n",
    "        FN +=1\n",
    "    if (i!=0 and j==0):\n",
    "        F_det +=1\n",
    "    if (j!=0 and i!=j):\n",
    "        FP_v += 1\n",
    "        \n",
    "print(\"TP: %d  \\nFN: %d \\nFP_v: %d\\nF_det: %d \" % (TP, FN, FP_v, F_det))\n",
    "\n",
    "FP = F_det + FP_v\n",
    "\n",
    "precision = TP/(TP+FP)\n",
    "recall = TP/(TP+FN)\n",
    "\n",
    "print(\"precision= %f \\nrecall= %f\"%(precision, recall))\n",
    "\n",
    "\n",
    "f1 = 2*(precision*recall)/(precision+recall)\n",
    "\n",
    "print(\"f1-score = %f\"%(f1))\n",
    "\n",
    "\n",
    "accuracy = TP/(TP + FN + FP)\n",
    "print(\"accuracy = %f\" % (accuracy))\n",
    "\n",
    "\n",
    "cm = confusion_matrix(y_true, y_pre)\n",
    "print(cm)\n",
    "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "plt.figure(figsize=(10,10))\n",
    "sbn.heatmap(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 510  \n",
      "FN: 64 \n",
      "FP_v: 134\n",
      "F_det: 113 \n",
      "precision= 0.673712 \n",
      "recall= 0.888502\n",
      "f1-score = 0.766341\n",
      "accuracy = 0.621194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ipykernel/__main__.py:34: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fb83758b3c8>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAJDCAYAAADKCVQfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm4ZWV55/3vr5ihmNGSKUACGm3Ni1IhphMUhwQ0vpChjUi6UUOsDO2QOaTNpWKu2JJEbdrXqeJsFINoKyHEIQmltESkVEAmFYlKAQUCpVhQCMW53z/2Lj11qL3XGdYez/fDtS72WWuv+9znsNh1172e51mpKiRJksbVilEnIEmS1I/FiiRJGmsWK5IkaaxZrEiSpLFmsSJJksaaxYokSRprFiuSJKk1Sd6V5I4k1/Q4niT/O8mNSa5O8qSmmBYrkiSpTe8BTu5z/FnAMd1tDfDWpoAWK5IkqTVV9Vng7j5vORV4X3V8HtgvycH9YlqsSJKkYToUuHnW1xu6+3raeaDpAIcd8PhW1vO/87572ggDwMpdd28t1qF7HtRKnI1b+hWhC3PXlu+3EufAPfZuJQ60l9O42nnFTq3EafPa/O7997YWq61roc3roK3f+daZh1qJM65WJK3Fmmnp8SznHXhiK3EAnn/XutZitfmZd/v3bmjvFz8PD95509CenbPrI37it+ncvtlmbVWtHeT3HHixIkmSpke3MFlKcXILcPisrw/r7uvJ20CSJGmYLgTO6M4KejLwvaq6rd8JdlYkSZp0Y3Q7M8l5wInAQUk2AK8CdgGoqrcBFwPPBm4E7gNe1BTTYkWSJLWmqp7fcLyA/76QmBYrkiRNupoZdQYD5ZgVSZI01uysSJI06WbsrEiSJI1MY2clyU/SWRp32+pytwAXVtX1g0xMkiTNTy3nMStJ/gz4EBDgC90twHlJzhp8epIkablr6qycCfynqnpw9s4kbwCuBV43qMQkSdI8LfMxKzPAITvYf3D32A4lWZNkfZL19/6gvWfeSJKk5aeps/L7wL8m+To/ekLijwFHAy/pddLs5wa09SBDSZLUw5SPWelbrFTVJ5I8Gjie7QfYXlFV47O2ryRJmlqNs4GqM8T480PIRZIk6WFcFE6SpEk3Rg8yHAQXhZMkSWPNzookSZNuygfY2lmRJEljzc6KJEmTbpkvCidJkjRSA++s/OChB5vfNA+bzvu9VuIA7P28N7UWK0krcTZt2dxKnDbN4Hp+87W1pZH4373/3lbitO2uLd8fdQoP09bvfNrN1Pj9f/z8u9aNOoUdGsfrfL6W9YMMJUmSRs0xK5IkTTrHrEiSJI2OnRVJkiadY1YkSZJGx86KJEmTbspnx9lZkSRJY83OiiRJk84xK5IkSaNjsSJJksbaom8DJXlRVb27zWQkSdIiuChcT2f3OpBkTZL1Sdbf/8B3l/AtJEnScte3s5Lk6l6HgFW9zquqtcBagEfs+5jxe4qWJEnTZMoH2DbdBloFnARsmrM/wGUDyUiSJGmWpmLlImBlVV0590CSdQPJSJIkLcyUj1npW6xU1Zl9jp3efjqSJEnbc1E4SZImXJXL7UuSJI2MnRVJkibdlM8GsrMiSZLGmp0VSZIm3ZTPBrKzIkmSxtrAOyurdt+/lTh7P+9NrcQB+P6HX95arL2fe25rscbNpi2bR52CxsReu+7eSpx7H7i/lTiS5nDMiiRJ0ug4ZkWSpEk34zorkiRJI2OxIkmSxpq3gSRJmnQOsJUkSRodOyuSJE06F4WTJEkancZiJclPJnlGkpVz9p88uLQkSdK81czwthHoW6wkeRnwceClwDVJTp11+LWDTEySJAmax6y8GDiuqjYnORK4IMmRVXUukEEnJ0mS5mHKx6w0FSsrqmozQFV9M8mJdAqWI+hTrCRZA6wBOHjvozhgj0e2lK4kSVpumsas3J7k2G1fdAuX5wAHAU/odVJVra2q1VW12kJFkqQBm5kZ3jYCTcXKGcDG2TuqamtVnQE8ZWBZSZIkdfW9DVRVG/oc+1z76UiSpIWq8kGGkiRJI+MKtpIkTbopnw1kZ0WSJI01OyuSJE06n7osSZI0OhYrkiRprHkbSJKkSTflA2wHXqzc8+C9g/4WC7b3c89tLdb3//HPW4mz36nntBIH4KGWLto9d9mtlTgA9299oLVYM1WtxdL83PvA/aNO4WHaejiZV5M0/uysSJI06RxgK0mSNDp2ViRJmnRTPmbFzookSRprdlYkSZp0jlmRJEkaHTsrkiRNOsesSJIkjU5jZyXJ8UBV1RVJHgecDNxQVRcPPDtJktRsyjsrfYuVJK8CngXsnOTTwM8AlwBnJXliVf3VEHKUJEnLWFNn5b8AxwK7ARuBw6rqniR/C1wO7LBYSbIGWAOw/56HsHK3A9rLWJIkbW+ZzwbaWlUPVdV9wDeq6h6AqtoC9PzNVNXaqlpdVastVCRJ0lI0dVYeSLJnt1g5btvOJPvSp1iRJElDtJzHrABPqaofAFRt12PaBXjBwLKSJEnq6lusbCtUdrD/TuDOgWQkSZI0i4vCSZI06Zb5AFtJkqSRsrMiSdKkm/IBtnZWJEnSWLOzIknSpHPMiiRJ0ugMvLNy4K77tBLnNu5uJQ5AtRYJjj7tra3E2fTGX24lDsA+L/9oK3F+8NCDrcQBmKn2futpKU6b14GGz/9+0iyOWZEkSRodx6xIkjTp7KxIkiSNjp0VSZImXYvjAseRnRVJkjTW7KxIkjTpHLMiSZI0OnZWJEmadHZWtpfkfYNIRJIkaUf6dlaSXDh3F/C0JPsBVNUpg0pMkiTN05Q/G6jpNtBhwHXAO+isbh1gNfD6AeclSZIENN8GWg18EXgF8L2qWgdsqarPVNVnep2UZE2S9UnW33nfxvaylSRJy07fzkpVzQBvTPLh7r9vbzqne95aYC3AEx/1c9O9Uo0kSaM25QNs5zUbqKo2AM9N8kvAPYNNSZIk6UcWNBuoqv6pqv7HoJKRJEmLUDW8rUGSk5N8NcmNSc7awfEfS3JJki8nuTrJs5tiuiicJElqRZKdgDcDzwIeBzw/yePmvO0vgPOr6onAacBbmuK6KJwkSZNufMasHA/cWFU3AST5EHAqnZnF2xSwT/f1vsCtTUEtViRJUlsOBW6e9fUG4GfmvOfVwKeSvBTYC3hmU1BvA0mSNOlmZoa2zV6epLutWWC2zwfeU1WHAc8G3p+kbz1iZ0WSJM3b7OVJduAW4PBZXx/W3TfbmcDJ3Vj/nmR34CDgjl7f086KJEmTrmaGt/V3BXBMkqOS7EpnAO3cR/d8G3gGQJLHArsD3+kX1GJFkiS1oqq2Ai8BPglcT2fWz7VJXpNk2/ME/wh4cZKrgPOAF1b1nxM98NtAu6xo51vsuevurcRp2+33freVOPu8/KOtxAG4+0WPbyXOAe++ppU4bWtrSeS9Wrym7n3g/tZiaX4etXL/VuJs3LyplTjSKNXM+CwWX1UXAxfP2ffKWa+vA35uITHtrEiSpLHmAFtJkibd+KyzMhB2ViRJ0lizsyJJ0qRrnqUz0eysSJKksWaxIkmSxpq3gSRJmnRjNHV5EOysSJKksbagzkqSn6fz+OdrqupTg0lJkiQtyHKeupzkC7Nevxj4/4C9gVclOWvAuUmSJDV2VnaZ9XoN8AtV9Z0kfwt8HnjdwDKTJEnzs5w7K8CKJPsnORBIVX0HoKruBbb2OinJmiTrk6y/477bWkxXkiQtN02dlX2BLwIBKsnBVXVbkpXdfTtUVWuBtQDHH/LU6R6iLEnSqPV/aPHE61usVNWRPQ7NAL/SejaSJElzLGqdlaq6D/iPlnORJEmLsczHrEiSJI2UK9hKkjTpXMFWkiRpdOysSJI06coxK5IkSSNjZ0WSpEnnmBVJkqTRGXhnpWin2rv3gftbiQOw04r2arSD9tynlTgPPNTz6QUL9oj3XtdKnLue/5OtxAE46qPfbi3WPT+4r5U4bV5TGr6NmzeNOoWJ0Obn3UNTvpaHxpe3gSRJmnA15YWkt4EkSdJYs7MiSdKkc4CtJEnS6NhZkSRp0rkonCRJ0ujYWZEkadI5ZkWSJGl0+nZWkvwMcH1V3ZNkD+As4EnAdcBrq+p7Q8hRkiT1s8zXWXkXsG250HOBfYFzuvvePcC8JEmSgOYxKyuqats68Kur6knd1/83yZW9TkqyBlgDcMS+R/OIPQ9eeqaSJGnHlvmYlWuSvKj7+qokqwGSPBp4sNdJVbW2qlZX1WoLFUmStBRNnZXfAs5N8hfAncC/J7kZuLl7TJIkjdqUr7PSt1jpDqB9YZJ9gKO6799QVbcPIzlJkqR5rbNSVfcAVw04F0mStBjLfMyKJEnSSFmsSJKkseZy+5IkTbha5ovCSZIkjZSdFUmSJp0DbCVJkkZn4J2VL9/5jUF/iwV7qMV7e3fed09rscbNgefd0FqsLbde2lqsPQ45obVY0rRr8/NOY8zOiiRJ0ug4ZkWSpEk35cvt21mRJEljzc6KJEmTzjErkiRJo2NnRZKkCVd2ViRJkkbHzookSZNuOXdWkrwsyeHDSkaSJGmupttAfwlcnuTSJL+X5BHDSEqSJC3AzMzwthFoKlZuAg6jU7QcB1yX5BNJXpBk74FnJ0mSlr2mYqWqaqaqPlVVZwKHAG8BTqZTyOxQkjVJ1idZPzNzb4vpSpKk5aZpgG1mf1FVDwIXAhcm2bPXSVW1FlgLsPOuh073qB9JkkZtOQ+wBZ7X60BV3ddyLpIkSQ/Tt7NSVV8bViKSJGmRlnlnRZIkaaRcFE6SpAlXZWdFkiRpZOysSJI06RyzIkmSNDp2ViRJmnR2ViRJkkZn4J2VR63cv5U4GzdvaiUOtJcTwHP3eXwrcT73wG2txAG49f67WovVlj0OOaG1WKccfFwrcb7w/Z5PjFiwNq9PSVqosrMiSZI0Oo5ZkSRp0tlZkSRJGh07K5IkTbqZUScwWHZWJEnSWLNYkSRJY83bQJIkTTinLkuSJI1Q385Kkl2B04Bbq+pfkpwO/GfgemBtVT04hBwlSVI/U95ZaboN9O7ue/ZM8gJgJfBR4BnA8cALBpueJEla7pqKlSdU1U8l2Rm4BTikqh5K8vfAVYNPT5IkNVrmU5dXdG8F7Q3sCezb3b8bsEuvk5KsSbI+yfp7f3B3O5lKkqRlqamz8k7gBmAn4BXAh5PcBDwZ+FCvk6pqLbAW4LADHj/dN9IkSRqxaZ8N1LdYqao3JvmH7utbk7wPeCbwd1X1hWEkKEmSlrfGdVaq6tZZr78LXDDQjCRJ0sIs8zErkiRJI+UKtpIkTbhpH7NiZ0WSJI01OyuSJE06x6xIkiSNjp0VSZImXNlZkSRJGp2Bd1Y2bt406G+xYG3m9KbNl7YWa9y86xFPay3Wb26+pLVYD7b0V4jX7nFsK3Gg3Z9PkrQ9bwNJkjTpvA0kSZI0OnZWJEmacA6wlSRJGiE7K5IkTTo7K5IkSaNjZ0WSpAnnmBVJkqQRauysJPlx4FeBw4GHgK8BH6yqewacmyRJmodl3VlJ8jLgbcDuwE8Du9EpWj6f5MSBZydJkiZKkpOTfDXJjUnO6vGeX09yXZJrk3ywKWZTZ+XFwLFV9VCSNwAXV9WJSd4OfBx4Yo8k1gBrALLTvqxYsVdTHpIkaZHGpbOSZCfgzcAvABuAK5JcWFXXzXrPMcCfAz9XVZuSPLIp7nzGrGwraHYDVgJU1beBXXqdUFVrq2p1Va22UJEkadk4Hrixqm6qqgeADwGnznnPi4E3V9UmgKq6oyloU2flHXSqosuBE4BzAJI8Arh7YflLkqSBqIw6g20OBW6e9fUG4GfmvOfRAEk+B+wEvLqqPtEvaN9iparOTfIvwGOB11fVDd393wGesqD0JUnSxJs91KNrbVWtXUCInYFjgBOBw4DPJnlCVX233wl9VdW1wLULSEKSJA3RMMesdAuTXsXJLXQm4mxzWHffbBuAy6vqQeA/knyNTvFyRa/v6TorkiSpLVcAxyQ5KsmuwGnAhXPe8zE6XRWSHETnttBN/YJarEiSpFZU1VbgJcAngeuB86vq2iSvSXJK922fBO5Kch1wCfAnVXVXv7guty9J0oSrmbEZYEtVXQxcPGffK2e9LuAPu9u82FmRJEljzc6KJEkTblwWhRsUOyuSJGmspXPraHB23vXQwX4DaQxsufXSVuLsccgJrcSRNFpbH7hlqINIbvnZpw/tz9pD//3fhj5Axs6KJEkaa45ZkSRpwjlmRZIkaYTsrEiSNOHGaZ2VQbCzIkmSxpqdFUmSJtyAJ/aOnJ0VSZI01uysSJI04RyzIkmSNEJ2ViRJmnB2ViRJkkaob7GSZN8kr0tyQ5K7k9yV5Pruvv36nLcmyfok62dm7m0/a0mStGw0dVbOBzYBJ1bVAVV1IPC07r7ze51UVWuranVVrV6xYq/2spUkSQ9TNbxtFJqKlSOr6pyq2rhtR1VtrKpzgCMGm5okSVLzANtvJflT4L1VdTtAklXAC4GbB5ybJEmah+U+wPZ5wIHAZ7pjVu4G1gEHAM8dcG6SJEn9OytVtQn4s+62nSQvAt49oLwkSdI8VS3vzko/Z7eWhSRJUg99OytJru51CFjVfjqSJGmhambUGQxW0wDbVcBJdKYqzxbgsoFkJEmSNEtTsXIRsLKqrpx7IMm6gWQkSZIWZGbKx6w0DbA9s8+x09tPR5IkaXs+yFBqwR6HnNBKnC23XtpKHGgvJ0njz9lAkiRJI2RnRZKkCbfcV7CVJEkaKTsrkiRNuFE9DXlY7KxIkqSxZrEiSZLGmreBJEmacA6wlSRJGiE7K5IkTbhpX27fzookSRpri+6sJPnnqnpWm8lIkqSFm/bl9vsWK0me1OsQcGyf89YAawCy076sWLHXohOUJEnLW1Nn5QrgM3SKk7n263VSVa0F1gLsvOuhU75UjSRJozXti8I1FSvXA79dVV+feyDJzYNJSZIk6UeaipVX03sQ7kvbTUWSJC3GtM8G6lusVNUFfQ7v33IukiRJD7OUqctnt5aFJElatKoMbRuFptlAV/c6BKxqPx1JkqTtNY1ZWQWcBGyasz/AZQPJSJIkLchynw10EbCyqq6ceyDJuoFkJEmSNEvTANsz+xw7vf10JEnSQk37bCCfDSRJksaaT12Wxsgeh5zQWqwtt17aWqw285LUvml/NpCdFUmSNNYsViRJ0ljzNpAkSRPOAbaSJEkjZGdFkqQJN+VrwtlZkSRJ483OiiRJE84xK5IkSSNkZ0WSpAnnonCSJEkj1LdYSbJPkv+Z5P1JTp9z7C19zluTZH2S9TMz97aVqyRJ2oGZIW6j0NRZeTcQ4CPAaUk+kmS37rEn9zqpqtZW1eqqWr1ixV4tpSpJkpajpjErP1FVv9Z9/bEkrwD+LckpA85LkiTNUzHdY1aaipXdkqyoqhmAqvqrJLcAnwVWDjw7SZK07DXdBvpH4Omzd1TVe4A/Ah4YUE6SJGkBZmp42yj07axU1Z/22P+JJK8dTEqSJEk/spSpy2e3loUkSVq0GTK0bRT6dlaSXN3rELCq/XQkSZK21zTAdhVwErBpzv4Alw0kI0mSpFmaipWLgJVVdeXcA0nWDSQjSZK0IMt66nJVndnn2Om9jkmSJLXFBxlKU2qPQ05oLdaWWy9tJU6bOUn6kVEtgz8sPshQkiSNNTsrkiRNuGkfs2JnRZIkjTU7K5IkTTjHrEiSJI2QnRVJkiacnRVJkqQRsrMiSdKEczaQJEnSCNlZkSRpws1Md2PFzookSRpvfYuVJI9K8tYkb05yYJJXJ/lKkvOTHNznvDVJ1idZPzNzb/tZS5KkH5ohQ9tGoamz8h7gOuBm4BJgC/Bs4FLgbb1Oqqq1VbW6qlavWLFXS6lKkqTlqKlYWVVVb6qq1wH7VdU5VXVzVb0JOGII+UmSpGWuaYDt7GLmfXOO7dRyLpIkaRFq1AkMWFNn5eNJVgJU1V9s25nkaOCrg0xMkiQJGjorVfXKHvtvTPJPg0lJkiQthMvt93Z2a1lIkiT10LezkuTqXoeAVe2nI0mSFmom070qXNMA21XAScCmOfsDXDaQjCRJkmZpKlYuAlZW1ZVzDyRZN5CMJEnSgkz7bKCmAbZn9jl2evvpSJIkbW/gDzLccuulrcTZ45ATWokjaeH8/08ab84GkiRJGqGBd1YkSdJgzUz3ZCA7K5IkabzZWZEkacLNMN2tFTsrkiSpNUlOTvLVJDcmOavP+34tSSVZ3RTTYkWSpAlXQ9z6SbIT8GbgWcDjgOcnedwO3rc38HLg8vn8fBYrkiSpLccDN1bVTVX1APAh4NQdvO8vgXOA++cT1GJFkiS15VDg5llfb+ju+6EkTwIOr6p/mm9QB9hKkjThhjl1OckaYM2sXWurau08z10BvAF44UK+54KLlSSPrKo7FnqeJEmafN3CpFdxcgtw+KyvD+vu22Zv4PHAunSeFP0o4MIkp1TV+l7fs2+xkuSAubuALyR5IpCqurvf+ZIkafDGaLn9K4BjkhxFp0g5DfjhswSr6nvAQdu+7j4U+Y/7FSrQPGblTuCLs7b1dO49fan7eoeSrEmyPsn6d7zvvIZvIUmSpkFVbQVeAnwSuB44v6quTfKaJKcsNm7TbaA/AX4B+JOq+gpAkv+oqqMakv1hi+jBO2+a9idXS5I0UuP0B21VXQxcPGffK3u898T5xOzbWamq1wO/BbwyyRu686LH6XciSZKmXOMA26raADy32775NLDnwLOSJEnz5oMMu6rqQuBpwDMBkrxoUElJkiRts6BF4apqS1Vd0/3y7AHkI0mSFmhmiNsoNE1dvrrXIWBV++lIkiRtr2nMyirgJGDTnP0BLhtIRpIkaUHGaJ2VgWgqVi4CVlbVlXMPdBdykSRJGqi+xUpVndnn2Om9jkmSpOGpKZ8NNPAHGf7cT43fpKFHrdx/1Ck8zMbNc++0Ld6TDjq6lThfuvPGVuJIg3DTT/1kK3F+/OobWokjaXB86rIkSRNu2sesLGjqsiRJ0rBZrEiSpLHmbSBJkiact4EkSZJGyM6KJEkTrkadwIDZWZEkSWPNzookSRNuZsoXhbOzIkmSxpqdFUmSJpyzgSRJkkaob7GS5ORZr/dN8s4kVyf5YJJVfc5bk2R9kvV33Hdbm/lKkqQ5Zoa4jUJTZ+W1s16/HrgN+H+BK4C39zqpqtZW1eqqWv3IPQ9eepaSJGnZWsiYldVVdWz39RuTvGAQCUmSpIWZ9nVWmoqVRyb5QyDAPklSVdt+J453kSRJA9dUrPwdsHf39XuBg4DvJHkUcOUgE5MkSfMz7eus9C1WqursHvs3JrlkMClJkiT9yFJu5eywkJEkScM17bOB+nZWklzd6xDQc+qyJElSW5rGrKwCTgI2zdkf4LKBZCRJkjRLU7FyEbCyqh42mDbJuoFkJEmSFmRZT12uqjP7HDu9/XQkSZK2N/AHGX7pzhsH/S0WbOPmuXe1pss4/s6ltv341Te0EmfLrZe2Egdgj0NOaC2WtBAzU95bcWE3SZI01gbeWZEkSYM1qinFw2JnRZIkjTU7K5IkTbjpHrFiZ0WSJI05OyuSJE04x6xIkiSNkJ0VSZIm3ExGncFg2VmRJEljzc6KJEkTzhVs50hy4CASkSRJ2pG+xUqS1yU5qPt6dZKbgMuTfCvJU/uctybJ+iTrZ2bubTllSZI0Ww1xG4WmzsovVdWd3dd/Azyvqo4GfgF4fa+TqmptVa2uqtUrVuzVUqqSJGk5aipWdk6ybVzLHlV1BUBVfQ3YbaCZSZIk0TzA9i3AxUleB3wiybnAR4GnA1cOOjlJktRs2heF61usVNWbknwF+F3g0d33HwN8DPjLwacnSZKWu8apy1W1Dlg3d3+SFwHvbj8lSZK0EE5d7u3s1rKQJEnqoW9nJcnVvQ4Bq9pPR5IkLdR091WabwOtAk4CNs3ZH+CygWQkSZI0S1OxchGwsqoeNvMnybqBZCRJkhZkuc8GOrPPsdPbT0eSJGl7A3+Q4dkHn9hKnFfdtq6VOJI02x6HnNBarLt+47GtxTrwA9e3FkvTz9lAkiRJIzTwzookSRqs6e6r2FmRJEljzs6KJEkTbtpnA9lZkSRJY83OiiRJE66mfNSKnRVJkjTWLFYkSdJY8zaQJEkTzgG2kiRJI9S3s5LkS8BHgfOq6hvDSUmSJC3Ecl9uf39gP+CSJF9I8gdJDmkKmmRNkvVJ1l+x+cZWEpUkSctTU7Gyqar+uKp+DPgj4BjgS0kuSbKm10lVtbaqVlfV6p9eeXSb+UqSpDlqiNsozHvMSlVdWlW/BxwKnAP87MCykiRJ6mqaDfS1uTuq6iHgE91NkiSN2LIes1JVp/U6luRF7acjSZK0vaVMXT67tSwkSdKizQxxG4WmqctX9zoErGo/HUmSpO01jVlZBZwEbJqzP8BlA8lIkiQtyLQ/yLCpWLkIWFlVV849kGTdQDKSJEmapW+xUlVn9jl2evvpSJKkhZr2ZwP5IENJasmBH7i+tVh3/cZjW4nTZk7SqFisSJI04aZ9zIpPXZYkSWPNYkWSJI01bwNJkjThpn2ArZ0VSZI01uysSJI04WbKAbaSJEkjY2dFkqQJN919FTsrkiRpzNlZkSRpws1MeW/FzookSRprfYuVJKuTXJLk75McnuTTSb6X5IokT+xz3pok65Osv2Lzje1nLUmSfqiG+M8oNHVW3gL8NfBPwGXA26tqX+Cs7rEdqqq1VbW6qlb/9MqjW0tWkiQtP03Fyi5V9c9VdR5QVXUBnRf/Cuw+8OwkSVKjmSFuo9BUrNyf5BeTPBeoJL8MkOSpwEMDz06SJC17TbOBfofObaAZ4CTgd5O8B7gFePFgU5MkSfOxrGcDVdVVVXVSVT2rqm6oqpdX1X5V9Z+AxwwpR0mStIwtZery2a1lIUmSFm3aZwP1vQ2U5Opeh4BV7acjSZK0vaYxK6vojFXZNGd/6ExlliRJGqimYuUiYGVVXTn3QJJ1A8lIkiQtyKimFA9L32Klqs7sc+z09tORJEna3sAfZPiSp9/eSpxXfaCVMFqAsw8+sbVYr7ptXWuxpOXgwA9cP+oUNEGqlvHUZUmSpIVIcnIFduZ5AAASP0lEQVSSrya5MclZOzj+h0muS3J1kn9NckRTTIsVSZIm3Aw1tK2fJDsBbwaeBTwOeH6Sx81525eB1VX1U8AFdBaf7ctiRZIkteV44MaquqmqHgA+BJw6+w1VdUlV3df98vPAYU1BBz5mRZIkDdYYzQY6FLh51tcbgJ/p8/4zgX9uCmqxIkmS5i3JGmDNrF1rq2rtIuL8V2A18NSm91qsSJI04Ya5DH63MOlVnNwCHD7r68O6+7aT5JnAK4CnVtUPmr6nY1YkSVJbrgCOSXJUkl2B04ALZ78hyROBtwOnVNUd8wlqZ0WSpAnXNEtnWKpqa5KXAJ8EdgLeVVXXJnkNsL6qLgT+BlgJfDgJwLer6pR+cS1WJElSa6rqYuDiOfteOev1Mxca02JFkqQJ5wq2kiRJI9S3WEmyMslrklyb5HtJvpPk80le2HDemiTrk6x/z1cfNghYkiS1aGaI2yg0dVY+ANwEnAScDfxv4L8BT0vy2l4nVdXaqlpdVatf+JhDW0tWkiQtP03FypFV9Z6q2lBVb6AzzejrwIuAXx18epIkqUkN8Z9RaCpW7k3y8wBJTgHuBqiqGSADzk2SJKlxNtDvAO9IcgxwLfCbAEkeQeepipIkSQPVt1ipqqvpPEFx7v7vJPn+wLKSJEnzNi6Lwg3KUqYun91aFpIkST307awkubrXIWBV++lIkqSFmvZF4ZrGrKyiM21505z9AS4bSEaSJEmzNBUrFwErq+rKuQeSrBtIRpIkaUGmfcxK0wDbM/scO739dCRJkrY38AcZHviB6wf9LTQgJz10b2uxXtVaJKldn3/kT7cW68l3XNFaLGkhRrVY27D4IENJkjTWBt5ZkSRJgzUz5bOB7KxIkqSxZmdFkqQJN919FTsrkiRpzNlZkSRpwk37Oit2ViRJ0lizsyJJ0oSzsyJJkjRCfYuVJPsmeV2SG5LcneSuJNd39+03rCQlSdLy1dRZOZ/OE5dPrKoDqupA4GndfecPOjlJktSsqoa2jUJTsXJkVZ1TVRu37aiqjVV1DnBEr5OSrEmyPsn6mZn2ni8jSZKWn6Zi5VtJ/jTJqm07kqxK8mfAzb1Oqqq1VbW6qlavWLFXW7lKkqQdmKGGto1CU7HyPOBA4DNJNiW5G1gHHAD8+oBzkyRJ6j91uao2JXk38Gng81W1eduxJCcDnxhwfpIkqUEt56nLSV4GfBx4CXBNklNnHX7tIBOTJEmC5kXhXgwcV1WbkxwJXJDkyKo6F8igk5MkSc1GNUtnWJqKlRXbbv1U1TeTnEinYDkCixVJkjQETQNsb09y7LYvuoXLc4CDgCcMMjFJkjQ/y3020BnAxtk7qmprVZ0BPGVgWUmSJHU1zQba0OfY59pPR5IkLdRyH7OiZezJd1wx6hSkgfM6H74tt17aWqw9DjmhtVgaXxYrkiRNuFGNJRmWpjErkiRJI2VnRZKkCbesV7CVJEkaNYsVSZI01rwNJEnShJuZ8qnLdlYkSdJYs7MiSdKEc4CtJEnSCNlZkSRpwjlmpYck/9xmIpIkSTvSt7OS5Em9DgHH9jlvDbAGIDvty4oVey06QUmS1N+0j1lpug10BfAZOsXJXPv1Oqmq1gJrAXbe9dDp/g1KkqSBaipWrgd+u6q+PvdAkpsHk5IkSVqI5T5m5dV93vPSdlORJEl6uL7FSlVdACTJM5KsnHP4/sGlJUmS5quG+M8o9C1WkrwM+DidLso1SU6ddfi1g0xMkiQJmsesvBg4rqo2JzkSuCDJkVV1LjsedCtJkoZs2sesNBUrK6pqM0BVfTPJiXQKliOwWJEkSUPQNMD29iQ/XE+lW7g8BzgIeMIgE5MkSfOzrMesAGcAG2fvqKqtVXUG8JSBZSVJktTV9zZQVW3oc+xz7acjSZK0PR9kKEkaqj0OOaG1WFtuvbS1WG3mNWxVM6NOYaAW/SBDSZKkYbCzIknShJuZ8gcZ2lmRJEljzc6KJEkTrqZ8UTg7K5IkaazZWZEkacI5ZkWSJGmE7KxIkjThHLMiSZI0QnZWJEmacDPLubOSZJ8k/zPJ+5OcPufYWwabmiRJUvNtoHcDAT4CnJbkI0l26x57cq+TkqxJsj7J+pmZe1tKVZIk7UgN8Z9RaCpWfqKqzqqqj1XVKcCXgH9LcmC/k6pqbVWtrqrVK1bs1VqykiRp+Wkas7JbkhXVfZxjVf1VkluAzwIrB56dJElqtNxnA/0j8PTZO6rqPcAfAQ8MKCdJkqQf6lusVNWfAhuSPCPJyln7PwG8bNDJSZIkNc0GeinwceClwDVJTp11+K8GmZgkSZqfGWpo2yg0jVlZAxxXVZuTHAlckOTIqjqXziwhSZKkgWoqVlZU1WaAqvpmkhPpFCxHYLEiSdJYWO4DbG9Pcuy2L7qFy3OAg4AnDDIxSZIkaO6snAFsnb2jqrYCZyR5+8CykiRJ8zbty+33LVaqakOfY59rPx1JkqTt+SBDSZIm3LSPWRl4sbL1gVsG/S0kSVoy/7waX3ZWJEmacKNa/2RYmmYDSZIkjZSdFUmSJty0j1mxsyJJksaanRVJkibctK+zYmdFkiSNNTsrkiRNuHI2kCRJ0uj0LVaSPCrJW5O8OcmBSV6d5CtJzk9y8LCSlCRJy1dTZ+U9wHXAzcAlwBbg2cClwNt6nZRkTZL1SdavXbu2pVQlSdKOzFQNbRuF9JubneTLVfXE7utvV9WPzTp2ZVUdO4/vMd030iRJergM85vtsccRQ/uzdsuWbw31Z4PmAbazOy/vm3Nsp5ZzkSRJi7DcF4X7eJKVAFX1F9t2Jjka+OogE5MkSYKG20AASX4SOBS4vKo2z9p/clV9Yh7fY7rLPUmSHm6ot0p22/3wof1Z+4P7bx76baCm2UAvBT4OvBS4Jsmpsw6/dpCJSZIkQfOYlTXAcVW1OcmRwAVJjqyqcxly1ShJknZs2sesNA6w3Xbrp6q+meREOgXLEVisSJKkIWgaYHt7kh9OT+4WLs8BDgKeMMjEJEnS/FTV0LYmSU5O8tUkNyY5awfHd0vyD93jl3fv3PTVVKycAWyc8wvZWlVnAE9pzFiSJC0bSXYC3gw8C3gc8Pwkj5vztjOBTVV1NPBG4JymuH2LlaraUFUbexz73HwSlyRJg1VD3BocD9xYVTdV1QPAh4BT57znVOC93dcXAM9I0ndoiQ8ylCRJbTmUziN6ttnQ3bfD91TVVuB7wIH9gg6jWMl8tiS/Pd/3DiPOtOc07T/fOOY07T/fOOY07T/fOOY07T/fAuIM1dYHbsmwttnP/+tuawb9841TZ6WtH7bNX9o059RmLHMafixzGn4scxp+rGnPaSJV1dqqWj1rm/3E4luAw2d9fVh3Hzt6T5KdgX2Bu/p9z3EqViRJ0mS7AjgmyVFJdgVOAy6c854LgRd0X/8X4N+qYZpR0zorkiRJ81JVW5O8BPgknQcev6uqrk3yGmB9VV0IvBN4f5IbgbvpFDR9jVOxsrb5LUON02asccypzVjmNPxY5jT8WOY0/FjTntNUqqqLgYvn7HvlrNf3A89dSMzGBxlKkiSNkmNWJEnSWBt5sdK0LO8C4rwryR1JrlliPocnuSTJdUmuTfLyJcTaPckXklzVjXX2EnPbKcmXk1y0xDjfTPKVJFcmWb/EWPsluSDJDUmuT/Kzi4jxmG4u27Z7kvz+EnL6g+7v+5ok5yXZfZFxXt6Nce1C89nR9ZjkgCSfTvL17r/3X0Ks53bzmkmyeglx/qb73+7qJP8nyX5LiPWX3ThXJvlUkkMWG2vWsT9KUkkOWmROr05yy6xr69lLySnJS7u/r2uT/PUic/qHWfl8M8mVi80pybFJPr/t/+Ukxy8h1v+T5N+7nw3/mGSfecTZ4eflQq/1PnEWc533irXga71PrEVd61qCYT5PYAfPF9gJ+Abw48CuwFXA4xYZ6ynAk4BrlpjTwcCTuq/3Br62hJwCrOy+3gW4HHjyEnL7Q+CDwEVL/Bm/CRzU0n/D9wK/1X29K7BfC9fERuCIRZ5/KPAfwB7dr88HXriIOI8HrgH2pDO261+Aoxdw/sOuR+CvgbO6r88CzllCrMcCjwHWAauXEOcXgZ27r89ZYk77zHr9MuBti43V3X84nUF635rP9dojp1cDf7yI//47ivW07nWwW/frRy72Z5t1/PXAK5eQ06eAZ3VfPxtYt4RYVwBP7b7+TeAv5xFnh5+XC73W+8RZzHXeK9aCr/U+sRZ1rbstfht1Z2U+y/LOS1V9ls6o4iWpqtuq6kvd198Hrufhq+/NN1ZV96nVdIqVXZjXasUPl+Qw4JeAdyzm/EFIsi+dD713AlTVA1X13SWGfQbwjar61hJi7Azskc78/T2BWxcR47HA5VV1X3VWWPwM8KvzPbnH9Th7ien3Ar+82FhVdX1VfXW++fSJ86nuzwfweTprIiw21j2zvtyLeV7rff7ffSPwpy3EWbAesX4XeF1V/aD7njuWklOSAL8OnLeEnArY1gHZl3le6z1iPRr4bPf1p4Ffm0ecXp+XC7rWe8VZ5HXeK9aCr/U+sRZ1rWvxRl2szGdZ3pFJciTwRDodkcXG2Knb5r0D+HRVLTbW/6LzwT2z2FxmKeBTSb6Ypa08eBTwHeDd6dyeekeSvZaY22nM88N7R6rqFuBvgW8DtwHfq6pPLSLUNcAJSQ5Msiedv7Ue3nBOk1VVdVv39UZg1RLjte03gX9eSoAkf5XkZuA3gFc2vb9PnFOBW6rqqqXk0/WSbsv+XfO99dbDo+lcE5cn+UySn15iXicAt1fV15cQ4/eBv+n+zv8W+PMlxLqWH/1l8bks8Hqf83m56Gu9jc/decRa8LU+N1Zb17rmZ9TFythKshL4CPD7c6roBamqh6rqWDpV/PFJHr+IXJ4D3FFVX1xsHnP8fFU9ic5TMf97ksU+QXtnOq3kt1bVE4F76bR8FyWdBYROAT68hBj70/nAPQo4BNgryX9daJyqup5Oq/hTwCeAK4GHFpvXDuLP85lgw5HkFcBW4ANLiVNVr6iqw7txXrLIXPYE/gft/AHwVuAngGPpFK+vX0KsnYEDgCcDfwKc3+2OLNbzWUJh3vW7wB90f+d/QLfLuUi/Cfxeki/SueXxwHxP7Pd5uZBrva3P3X6xFnOt7yhWG9e65m/Uxcp8luUduiS70LkwP1BVH20jZvf2yCXAyYs4/eeAU5J8k86tsqcn+fsl5HJL9993AP+Hzu24xdgAbJjVLbqATvGyWM8CvlRVty8hxjOB/6iq71TVg8BHgf+8mEBV9c6qOq6qngJsonO/eiluT3IwQPffjbcRhiHJC4HnAL/R/YOlDR9gHrcRevgJOsXmVd1r/jDgS0ketdBAVXV79y8MM8DfsfhrHTrX+0e7t3e/QKfL2Tjwd0e6tyh/FfiHJeQDnVVAt31GfZgl/HxVdUNV/WJVHUeniPrGfM7r8Xm54Gu9zc/dXrEWc63PI6+lXOuap1EXK/NZlneoun9TeidwfVW9YYmxHrFtxHmSPYBfAG5YaJyq+vOqOqyqjqTzO/q3qlpwt6Cbx15J9t72ms6gs0XNoKqqjcDNSR7T3fUM4LrFxOpq42+a3waenGTP7n/LZ9C5z7xgSR7Z/feP0fmD5YNLzG32EtMvAD6+xHhLluRkOrcXT6mq+5YY65hZX57KIq51gKr6SlU9sqqO7F7zG+gMcty4iJwOnvXlr7DIa73rY3QG2ZLk0XQGlN+5yFjPBG6oqg1LyAc6Y1Se2n39dGDRt5RmXe8rgL8A3jaPc3p9Xi7oWm/5c3eHsRZzrfeJ1cq1rgWoIY/onbvRGQvwNTpV/CuWEOc8Om3eB+l8uJ25yDg/T6dleTWd1v+VwLMXGeungC93Y13DPEf9N8Q8kSXMBqIz8+qq7nbtUn7n3XjHAuu7P+PHgP0XGWcvOg+y2reF39HZdD48rgHeT3f2xiLiXEqn+LoKeMZSr0c6j0D/Vzp/oPwLcMASYv1K9/UPgNuBTy4yzo10xo1tu9bnO4NnR7E+0v2dXw38I52BiIuKNef4N5nfbKAd5fR+4CvdnC4EDl7Cz7cr8Pfdn/FLwNMX+7MB7wF+p4Vr6ueBL3av0cuB45YQ6+V0Pou/BryO7qKhDXF2+Hm50Gu9T5zFXOe9Yi34Wu8Ta1HXutviN1ewlSRJY23Ut4EkSZL6sliRJEljzWJFkiSNNYsVSZI01ixWJEnSWLNYkSRJY81iRZIkjTWLFUmSNNb+f3HuaIHWBkWeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#### argument metrics for SRL\n",
    "\n",
    "y_true, y_pre = arg_real[mask],  arg_pred[mask]\n",
    "\n",
    "TP,FN , F_det, FP_v = 0,0,0,0\n",
    "for i,j in zip(y_pre, y_true):\n",
    "    if i==j and i!= 0:\n",
    "        TP += 1\n",
    "    if j!=0 and i==0:\n",
    "        FN +=1\n",
    "    if (i!=0 and j==0):\n",
    "        F_det +=1\n",
    "    if (j!=0 and i!=j):\n",
    "        FP_v += 1\n",
    "        \n",
    "print(\"TP: %d  \\nFN: %d \\nFP_v: %d\\nF_det: %d \" % (TP, FN, FP_v, F_det))\n",
    "\n",
    "FP = F_det + FP_v\n",
    "\n",
    "precision = TP/(TP+FP)\n",
    "recall = TP/(TP+FN)\n",
    "\n",
    "print(\"precision= %f \\nrecall= %f\"%(precision, recall))\n",
    "\n",
    "\n",
    "f1 = 2*(precision*recall)/(precision+recall)\n",
    "\n",
    "print(\"f1-score = %f\"%(f1))\n",
    "\n",
    "\n",
    "accuracy = TP/(TP + FN + FP)\n",
    "print(\"accuracy = %f\" % (accuracy))\n",
    "cm = confusion_matrix(y_true, y_pre)\n",
    "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "plt.figure(figsize=(10,10))\n",
    "sbn.heatmap(cm)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_tensorflow_p36",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}