src/relation_extraction/re.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "import pandas as pd\n",
    "import itertools\n",
    "import utils\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from random import Random\n",
    "from sklearn.metrics import classification_report\n",
    "import load_data\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example from sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = [[0], [1], [2], [3]]\n",
    "y = [0, 1, 2, 3]\n",
    "clf = svm.SVC(decision_function_shape='ovr')\n",
    "clf.fit(X, y)\n",
    "clf.predict([[5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parse Dia-Bete data\n",
    "Get list of relations from Dia-Bete file. The 'Relation' column is in the following format: causes|prevents, the name of the relation that this entity is the target separated by '|'. The 'Relation ID' column is in this format: 1-1[1_2]|1-19[3_2], where the first two numbers tell the ID of the first token of the source entity and the two numbers in square brackets say the entity number of the source entity and the target entity (the current entity).\n",
    "\n",
    "For each pair of words have a label if they are connected by a relation or not. Use word.isupper as the only feature for now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "505\n",
      "main_resposta_0180.txt\n",
      "nutrition_resposta_0219.txt\n",
      "nutrition_resposta_0104.txt\n",
      "nutrition_resposta_0103.txt\n",
      "main_resposta_0266.txt\n",
      "main_resposta_0148.txt\n",
      "nutrition_resposta_0075.txt\n",
      "main_resposta_0076.txt\n",
      "main_resposta_0161.txt\n",
      "main_resposta_0133.txt\n",
      "main_resposta_0097.txt\n",
      "main_resposta_0119.txt\n",
      "nutrition_resposta_0157.txt\n",
      "main_resposta_0267.txt\n",
      "nutrition_resposta_0095.txt\n",
      "nutrition_resposta_0212.txt\n",
      "nutrition_resposta_0141.txt\n",
      "main_resposta_0268.txt\n",
      "main_resposta_0191.txt\n",
      "main_resposta_0075.txt\n",
      "nutrition_resposta_0112.txt\n",
      "nutrition_resposta_0184.txt\n",
      "main_resposta_0214.txt\n",
      "nutrition_resposta_0246.txt\n",
      "nutrition_resposta_0237.txt\n",
      "nutrition_resposta_0010.txt\n",
      "nutrition_resposta_0187.txt\n",
      "nutrition_resposta_0211.txt\n",
      "nutrition_resposta_0051.txt\n",
      "main_resposta_0287.txt\n",
      "nutrition_resposta_0233.txt\n",
      "nutrition_resposta_0130.txt\n",
      "nutrition_resposta_0063.txt\n",
      "main_resposta_0252.txt\n",
      "nutrition_resposta_0037.txt\n",
      "nutrition_resposta_0139.txt\n",
      "nutrition_resposta_0071.txt\n",
      "main_resposta_0022.txt\n",
      "main_resposta_0222.txt\n",
      "main_resposta_0060.txt\n",
      "nutrition_resposta_0100.txt\n",
      "main_resposta_0087.txt\n",
      "nutrition_resposta_0099.txt\n",
      "main_resposta_0179.txt\n",
      "nutrition_resposta_0152.txt\n",
      "main_resposta_0262.txt\n",
      "nutrition_resposta_0173.txt\n",
      "main_resposta_0120.txt\n",
      "main_resposta_0178.txt\n",
      "nutrition_resposta_0021.txt\n",
      "main_resposta_0246.txt\n",
      "nutrition_resposta_0019.txt\n",
      "main_resposta_0199.txt\n",
      "main_resposta_0278.txt\n",
      "main_resposta_0132.txt\n",
      "main_resposta_0183.txt\n",
      "main_resposta_0298.txt\n",
      "main_resposta_0258.txt\n",
      "main_resposta_0281.txt\n",
      "nutrition_resposta_0131.txt\n",
      "main_resposta_0212.txt\n",
      "main_resposta_0165.txt\n",
      "nutrition_resposta_0105.txt\n",
      "main_resposta_0093.txt\n",
      "main_resposta_0091.txt\n",
      "main_resposta_0144.txt\n",
      "nutrition_resposta_0150.txt\n",
      "nutrition_resposta_0119.txt\n",
      "main_resposta_0238.txt\n",
      "nutrition_resposta_0125.txt\n",
      "main_resposta_0194.txt\n",
      "main_resposta_0069.txt\n",
      "main_resposta_0117.txt\n",
      "nutrition_resposta_0058.txt\n",
      "main_resposta_0294.txt\n",
      "main_resposta_0177.txt\n",
      "main_resposta_0009.txt\n",
      "nutrition_resposta_0061.txt\n",
      "nutrition_resposta_0111.txt\n",
      "main_resposta_0052.txt\n",
      "nutrition_resposta_0235.txt\n",
      "nutrition_resposta_0229.txt\n",
      "main_resposta_0089.txt\n",
      "main_resposta_0202.txt\n",
      "main_resposta_0163.txt\n",
      "nutrition_resposta_0222.txt\n",
      "nutrition_resposta_0166.txt\n",
      "nutrition_resposta_0060.txt\n",
      "nutrition_resposta_0073.txt\n",
      "main_resposta_0228.txt\n",
      "main_resposta_0005.txt\n",
      "nutrition_resposta_0004.txt\n",
      "main_resposta_0121.txt\n",
      "main_resposta_0185.txt\n",
      "main_resposta_0141.txt\n",
      "nutrition_resposta_0203.txt\n",
      "nutrition_resposta_0195.txt\n",
      "main_resposta_0216.txt\n",
      "nutrition_resposta_0143.txt\n",
      "main_resposta_0106.txt\n",
      "main_resposta_0101.txt\n",
      "main_resposta_0043.txt\n",
      "nutrition_resposta_0093.txt\n",
      "main_resposta_0107.txt\n",
      "nutrition_resposta_0027.txt\n",
      "nutrition_resposta_0258.txt\n",
      "main_resposta_0023.txt\n",
      "main_resposta_0283.txt\n",
      "nutrition_resposta_0175.txt\n",
      "main_resposta_0241.txt\n",
      "nutrition_resposta_0240.txt\n",
      "nutrition_resposta_0200.txt\n",
      "nutrition_resposta_0247.txt\n",
      "main_resposta_0259.txt\n",
      "main_resposta_0176.txt\n",
      "nutrition_resposta_0154.txt\n",
      "nutrition_resposta_0236.txt\n",
      "main_resposta_0290.txt\n",
      "nutrition_resposta_0226.txt\n",
      "main_resposta_0154.txt\n",
      "nutrition_resposta_0092.txt\n",
      "nutrition_resposta_0230.txt\n",
      "nutrition_resposta_0138.txt\n",
      "nutrition_resposta_0069.txt\n",
      "nutrition_resposta_0133.txt\n",
      "main_resposta_0046.txt\n",
      "nutrition_resposta_0036.txt\n",
      "nutrition_resposta_0177.txt\n",
      "nutrition_resposta_0144.txt\n",
      "nutrition_resposta_0106.txt\n",
      "nutrition_resposta_0254.txt\n",
      "nutrition_resposta_0035.txt\n",
      "main_resposta_0111.txt\n",
      "main_resposta_0038.txt\n",
      "nutrition_resposta_0001.txt\n",
      "nutrition_resposta_0072.txt\n",
      "nutrition_resposta_0031.txt\n",
      "main_resposta_0020.txt\n",
      "nutrition_resposta_0180.txt\n",
      "nutrition_resposta_0171.txt\n",
      "nutrition_resposta_0052.txt\n",
      "main_resposta_0114.txt\n",
      "main_resposta_0042.txt\n",
      "main_resposta_0156.txt\n",
      "nutrition_resposta_0186.txt\n",
      "main_resposta_0292.txt\n",
      "nutrition_resposta_0076.txt\n",
      "main_resposta_0012.txt\n",
      "nutrition_resposta_0134.txt\n",
      "nutrition_resposta_0178.txt\n",
      "main_resposta_0024.txt\n",
      "main_resposta_0014.txt\n",
      "nutrition_resposta_0059.txt\n",
      "nutrition_resposta_0185.txt\n",
      "main_resposta_0188.txt\n",
      "nutrition_resposta_0174.txt\n",
      "main_resposta_0303.txt\n",
      "nutrition_resposta_0188.txt\n",
      "main_resposta_0054.txt\n",
      "main_resposta_0155.txt\n",
      "main_resposta_0181.txt\n",
      "main_resposta_0070.txt\n",
      "main_resposta_0040.txt\n",
      "nutrition_resposta_0153.txt\n",
      "nutrition_resposta_0217.txt\n",
      "nutrition_resposta_0216.txt\n",
      "nutrition_resposta_0202.txt\n",
      "nutrition_resposta_0189.txt\n",
      "main_resposta_0027.txt\n",
      "main_resposta_0201.txt\n",
      "main_resposta_0142.txt\n",
      "main_resposta_0171.txt\n",
      "nutrition_resposta_0118.txt\n",
      "nutrition_resposta_0043.txt\n",
      "main_resposta_0168.txt\n",
      "main_resposta_0157.txt\n",
      "main_resposta_0078.txt\n",
      "main_resposta_0140.txt\n",
      "nutrition_resposta_0192.txt\n",
      "nutrition_resposta_0209.txt\n",
      "main_resposta_0073.txt\n",
      "nutrition_resposta_0228.txt\n",
      "main_resposta_0063.txt\n",
      "main_resposta_0231.txt\n",
      "nutrition_resposta_0251.txt\n",
      "nutrition_resposta_0242.txt\n",
      "main_resposta_0270.txt\n",
      "main_resposta_0130.txt\n",
      "nutrition_resposta_0088.txt\n",
      "nutrition_resposta_0163.txt\n",
      "nutrition_resposta_0215.txt\n",
      "main_resposta_0280.txt\n",
      "main_resposta_0215.txt\n",
      "nutrition_resposta_0108.txt\n",
      "nutrition_resposta_0044.txt\n",
      "main_resposta_0003.txt\n",
      "main_resposta_0019.txt\n",
      "main_resposta_0193.txt\n",
      "main_resposta_0208.txt\n",
      "nutrition_resposta_0023.txt\n",
      "main_resposta_0077.txt\n",
      "nutrition_resposta_0074.txt\n",
      "main_resposta_0210.txt\n",
      "nutrition_resposta_0162.txt\n",
      "nutrition_resposta_0082.txt\n",
      "nutrition_resposta_0012.txt\n",
      "nutrition_resposta_0038.txt\n",
      "main_resposta_0213.txt\n",
      "main_resposta_0184.txt\n",
      "main_resposta_0172.txt\n",
      "main_resposta_0053.txt\n",
      "main_resposta_0067.txt\n",
      "main_resposta_0255.txt\n",
      "main_resposta_0134.txt\n",
      "main_resposta_0145.txt\n",
      "main_resposta_0245.txt\n",
      "main_resposta_0169.txt\n",
      "nutrition_resposta_0198.txt\n",
      "main_resposta_0242.txt\n",
      "main_resposta_0159.txt\n",
      "nutrition_resposta_0033.txt\n",
      "nutrition_resposta_0039.txt\n",
      "nutrition_resposta_0132.txt\n",
      "main_resposta_0064.txt\n",
      "main_resposta_0146.txt\n",
      "main_resposta_0269.txt\n",
      "nutrition_resposta_0208.txt\n",
      "main_resposta_0061.txt\n",
      "nutrition_resposta_0168.txt\n",
      "main_resposta_0173.txt\n",
      "main_resposta_0263.txt\n",
      "nutrition_resposta_0227.txt\n",
      "main_resposta_0018.txt\n",
      "main_resposta_0279.txt\n",
      "main_resposta_0218.txt\n",
      "main_resposta_0021.txt\n",
      "main_resposta_0131.txt\n",
      "nutrition_resposta_0077.txt\n",
      "nutrition_resposta_0158.txt\n",
      "nutrition_resposta_0241.txt\n",
      "main_resposta_0219.txt\n",
      "main_resposta_0236.txt\n",
      "nutrition_resposta_0114.txt\n",
      "nutrition_resposta_0068.txt\n",
      "main_resposta_0150.txt\n",
      "main_resposta_0013.txt\n",
      "main_resposta_0254.txt\n",
      "nutrition_resposta_0098.txt\n",
      "nutrition_resposta_0042.txt\n",
      "main_resposta_0016.txt\n",
      "nutrition_resposta_0181.txt\n",
      "main_resposta_0189.txt\n",
      "main_resposta_0149.txt\n",
      "main_resposta_0015.txt\n",
      "main_resposta_0248.txt\n",
      "main_resposta_0112.txt\n",
      "main_resposta_0152.txt\n",
      "nutrition_resposta_0011.txt\n",
      "main_resposta_0080.txt\n",
      "main_resposta_0250.txt\n",
      "main_resposta_0289.txt\n",
      "main_resposta_0147.txt\n",
      "nutrition_resposta_0097.txt\n",
      "nutrition_resposta_0053.txt\n",
      "main_resposta_0200.txt\n",
      "nutrition_resposta_0096.txt\n",
      "nutrition_resposta_0182.txt\n",
      "nutrition_resposta_0054.txt\n",
      "nutrition_resposta_0085.txt\n",
      "nutrition_resposta_0164.txt\n",
      "nutrition_resposta_0014.txt\n",
      "main_resposta_0247.txt\n",
      "nutrition_resposta_0015.txt\n",
      "nutrition_resposta_0006.txt\n",
      "main_resposta_0086.txt\n",
      "nutrition_resposta_0091.txt\n",
      "main_resposta_0048.txt\n",
      "main_resposta_0008.txt\n",
      "nutrition_resposta_0013.txt\n",
      "nutrition_resposta_0257.txt\n",
      "nutrition_resposta_0140.txt\n",
      "nutrition_resposta_0055.txt\n",
      "main_resposta_0227.txt\n",
      "main_resposta_0039.txt\n",
      "main_resposta_0295.txt\n",
      "nutrition_resposta_0183.txt\n",
      "nutrition_resposta_0016.txt\n",
      "main_resposta_0094.txt\n",
      "main_resposta_0304.txt\n",
      "nutrition_resposta_0147.txt\n",
      "main_resposta_0230.txt\n",
      "nutrition_resposta_0107.txt\n",
      "nutrition_resposta_0005.txt\n",
      "main_resposta_0099.txt\n",
      "nutrition_resposta_0220.txt\n",
      "main_resposta_0056.txt\n",
      "nutrition_resposta_0057.txt\n",
      "main_resposta_0207.txt\n",
      "main_resposta_0032.txt\n",
      "main_resposta_0051.txt\n",
      "nutrition_resposta_0243.txt\n",
      "nutrition_resposta_0116.txt\n",
      "main_resposta_0211.txt\n",
      "nutrition_resposta_0050.txt\n",
      "nutrition_resposta_0199.txt\n",
      "nutrition_resposta_0231.txt\n",
      "main_resposta_0050.txt\n",
      "main_resposta_0049.txt\n",
      "main_resposta_0098.txt\n",
      "main_resposta_0209.txt\n",
      "nutrition_resposta_0009.txt\n",
      "main_resposta_0125.txt\n",
      "main_resposta_0243.txt\n",
      "main_resposta_0296.txt\n",
      "main_resposta_0293.txt\n",
      "main_resposta_0068.txt\n",
      "nutrition_resposta_0048.txt\n",
      "main_resposta_0110.txt\n",
      "nutrition_resposta_0062.txt\n",
      "nutrition_resposta_0206.txt\n",
      "main_resposta_0122.txt\n",
      "main_resposta_0264.txt\n",
      "nutrition_resposta_0151.txt\n",
      "main_resposta_0170.txt\n",
      "nutrition_resposta_0221.txt\n",
      "main_resposta_0035.txt\n",
      "nutrition_resposta_0030.txt\n",
      "nutrition_resposta_0248.txt\n",
      "nutrition_resposta_0020.txt\n",
      "nutrition_resposta_0191.txt\n",
      "main_resposta_0004.txt\n",
      "nutrition_resposta_0135.txt\n",
      "nutrition_resposta_0155.txt\n",
      "nutrition_resposta_0169.txt\n",
      "nutrition_resposta_0113.txt\n",
      "main_resposta_0302.txt\n",
      "main_resposta_0011.txt\n",
      "main_resposta_0174.txt\n",
      "nutrition_resposta_0120.txt\n",
      "main_resposta_0116.txt\n",
      "nutrition_resposta_0146.txt\n",
      "main_resposta_0062.txt\n",
      "main_resposta_0232.txt\n",
      "main_resposta_0265.txt\n",
      "main_resposta_0225.txt\n",
      "nutrition_resposta_0080.txt\n",
      "nutrition_resposta_0029.txt\n",
      "nutrition_resposta_0197.txt\n",
      "nutrition_resposta_0137.txt\n",
      "nutrition_resposta_0232.txt\n",
      "main_resposta_0103.txt\n",
      "nutrition_resposta_0159.txt\n",
      "nutrition_resposta_0239.txt\n",
      "nutrition_resposta_0122.txt\n",
      "nutrition_resposta_0007.txt\n",
      "nutrition_resposta_0115.txt\n",
      "nutrition_resposta_0148.txt\n",
      "main_resposta_0085.txt\n",
      "nutrition_resposta_0256.txt\n",
      "main_resposta_0244.txt\n",
      "nutrition_resposta_0034.txt\n",
      "main_resposta_0066.txt\n",
      "main_resposta_0065.txt\n",
      "main_resposta_0273.txt\n",
      "main_resposta_0036.txt\n",
      "main_resposta_0249.txt\n",
      "main_resposta_0044.txt\n",
      "main_resposta_0041.txt\n",
      "nutrition_resposta_0160.txt\n",
      "main_resposta_0124.txt\n",
      "main_resposta_0030.txt\n",
      "main_resposta_0123.txt\n",
      "main_resposta_0158.txt\n",
      "nutrition_resposta_0136.txt\n",
      "nutrition_resposta_0101.txt\n",
      "main_resposta_0010.txt\n",
      "nutrition_resposta_0255.txt\n",
      "main_resposta_0166.txt\n",
      "nutrition_resposta_0003.txt\n",
      "main_resposta_0001.txt\n",
      "main_resposta_0272.txt\n",
      "main_resposta_0277.txt\n",
      "main_resposta_0083.txt\n",
      "main_resposta_0192.txt\n",
      "nutrition_resposta_0127.txt\n",
      "main_resposta_0102.txt\n",
      "nutrition_resposta_0190.txt\n",
      "nutrition_resposta_0089.txt\n",
      "nutrition_resposta_0179.txt\n",
      "main_resposta_0153.txt\n",
      "nutrition_resposta_0017.txt\n",
      "main_resposta_0055.txt\n",
      "main_resposta_0260.txt\n",
      "main_resposta_0092.txt\n",
      "nutrition_resposta_0087.txt\n",
      "main_resposta_0257.txt\n",
      "nutrition_resposta_0142.txt\n",
      "nutrition_resposta_0065.txt\n",
      "nutrition_resposta_0110.txt\n",
      "main_resposta_0002.txt\n",
      "nutrition_resposta_0205.txt\n",
      "nutrition_resposta_0218.txt\n",
      "main_resposta_0223.txt\n",
      "main_resposta_0151.txt\n",
      "main_resposta_0058.txt\n",
      "main_resposta_0175.txt\n",
      "nutrition_resposta_0025.txt\n",
      "nutrition_resposta_0172.txt\n",
      "nutrition_resposta_0196.txt\n",
      "nutrition_resposta_0081.txt\n",
      "main_resposta_0275.txt\n",
      "nutrition_resposta_0193.txt\n",
      "main_resposta_0059.txt\n",
      "nutrition_resposta_0066.txt\n",
      "nutrition_resposta_0123.txt\n",
      "main_resposta_0006.txt\n",
      "main_resposta_0284.txt\n",
      "main_resposta_0129.txt\n",
      "nutrition_resposta_0223.txt\n",
      "main_resposta_0240.txt\n",
      "nutrition_resposta_0121.txt\n",
      "main_resposta_0196.txt\n",
      "main_resposta_0198.txt\n",
      "nutrition_resposta_0249.txt\n",
      "nutrition_resposta_0002.txt\n",
      "main_resposta_0217.txt\n",
      "nutrition_resposta_0234.txt\n",
      "nutrition_resposta_0102.txt\n",
      "nutrition_resposta_0210.txt\n",
      "nutrition_resposta_0018.txt\n",
      "main_resposta_0079.txt\n",
      "main_resposta_0057.txt\n",
      "main_resposta_0025.txt\n",
      "nutrition_resposta_0083.txt\n",
      "nutrition_resposta_0213.txt\n",
      "main_resposta_0253.txt\n",
      "main_resposta_0261.txt\n",
      "main_resposta_0071.txt\n",
      "nutrition_resposta_0194.txt\n",
      "main_resposta_0186.txt\n",
      "main_resposta_0205.txt\n",
      "main_resposta_0299.txt\n",
      "main_resposta_0220.txt\n",
      "main_resposta_0300.txt\n",
      "nutrition_resposta_0079.txt\n",
      "main_resposta_0135.txt\n",
      "main_resposta_0276.txt\n",
      "main_resposta_0288.txt\n",
      "nutrition_resposta_0078.txt\n",
      "main_resposta_0127.txt\n",
      "nutrition_resposta_0244.txt\n",
      "main_resposta_0136.txt\n",
      "main_resposta_0074.txt\n",
      "main_resposta_0235.txt\n",
      "nutrition_resposta_0026.txt\n",
      "nutrition_resposta_0149.txt\n",
      "main_resposta_0203.txt\n",
      "main_resposta_0256.txt\n",
      "nutrition_resposta_0032.txt\n",
      "nutrition_resposta_0161.txt\n",
      "nutrition_resposta_0165.txt\n",
      "nutrition_resposta_0225.txt\n",
      "nutrition_resposta_0145.txt\n",
      "main_resposta_0291.txt\n",
      "main_resposta_0109.txt\n",
      "main_resposta_0034.txt\n",
      "main_resposta_0138.txt\n",
      "main_resposta_0206.txt\n",
      "main_resposta_0162.txt\n",
      "nutrition_resposta_0109.txt\n",
      "main_resposta_0017.txt\n",
      "nutrition_resposta_0117.txt\n",
      "main_resposta_0118.txt\n",
      "main_resposta_0029.txt\n",
      "nutrition_resposta_0094.txt\n",
      "nutrition_resposta_0090.txt\n",
      "nutrition_resposta_0128.txt\n",
      "main_resposta_0167.txt\n",
      "nutrition_resposta_0047.txt\n",
      "main_resposta_0113.txt\n",
      "main_resposta_0286.txt\n",
      "nutrition_resposta_0224.txt\n",
      "nutrition_resposta_0024.txt\n",
      "nutrition_resposta_0170.txt\n",
      "main_resposta_0139.txt\n",
      "nutrition_resposta_0207.txt\n",
      "main_resposta_0237.txt\n",
      "nutrition_resposta_0201.txt\n",
      "main_resposta_0126.txt\n",
      "nutrition_resposta_0070.txt\n",
      "main_resposta_0274.txt\n",
      "main_resposta_0088.txt\n",
      "nutrition_resposta_0204.txt\n",
      "nutrition_resposta_0008.txt\n",
      "main_resposta_0037.txt\n",
      "nutrition_resposta_0028.txt\n",
      "nutrition_resposta_0046.txt\n",
      "nutrition_resposta_0176.txt\n",
      "nutrition_resposta_0056.txt\n",
      "main_resposta_0137.txt\n",
      "nutrition_resposta_0040.txt\n",
      "main_resposta_0108.txt\n",
      "main_resposta_0182.txt\n",
      "main_resposta_0190.txt\n",
      "main_resposta_0084.txt\n",
      "main_resposta_0187.txt\n",
      "nutrition_resposta_0124.txt\n",
      "nutrition_resposta_0022.txt\n",
      "main_resposta_0233.txt\n",
      "nutrition_resposta_0253.txt\n",
      "main_resposta_0195.txt\n",
      "main_resposta_0104.txt\n",
      "nutrition_resposta_0245.txt\n",
      "main_resposta_0082.txt\n",
      "main_resposta_0297.txt\n",
      "main_resposta_0234.txt\n",
      "main_resposta_0047.txt\n",
      "main_resposta_0072.txt\n",
      "main_resposta_0197.txt\n",
      "nutrition_resposta_0067.txt\n",
      "main_resposta_0301.txt\n",
      "main_resposta_0081.txt\n",
      "nutrition_resposta_0084.txt\n",
      "main_resposta_0128.txt\n",
      "nutrition_resposta_0167.txt\n",
      "main_resposta_0045.txt\n",
      "main_resposta_0271.txt\n",
      "nutrition_resposta_0049.txt\n",
      "nutrition_resposta_0064.txt\n",
      "main_resposta_0100.txt\n",
      "nutrition_resposta_0086.txt\n",
      "main_resposta_0095.txt\n",
      "nutrition_resposta_0252.txt\n",
      "nutrition_resposta_0041.txt\n",
      "main_resposta_0221.txt\n",
      "main_resposta_0160.txt\n",
      "nutrition_resposta_0238.txt\n",
      "nutrition_resposta_0045.txt\n",
      "nutrition_resposta_0250.txt\n",
      "main_resposta_0164.txt\n",
      "main_resposta_0007.txt\n",
      "main_resposta_0285.txt\n",
      "nutrition_resposta_0156.txt\n",
      "main_resposta_0226.txt\n",
      "nutrition_resposta_0126.txt\n",
      "main_resposta_0204.txt\n",
      "main_resposta_0028.txt\n",
      "nutrition_resposta_0214.txt\n",
      "main_resposta_0239.txt\n",
      "main_resposta_0224.txt\n",
      "main_resposta_0096.txt\n",
      "main_resposta_0031.txt\n",
      "main_resposta_0033.txt\n",
      "nutrition_resposta_0129.txt\n",
      "main_resposta_0090.txt\n",
      "main_resposta_0105.txt\n",
      "main_resposta_0143.txt\n",
      "main_resposta_0229.txt\n",
      "main_resposta_0251.txt\n",
      "main_resposta_0282.txt\n",
      "main_resposta_0026.txt\n",
      "main_resposta_0115.txt\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('../../data/all_annotated_dataset_2022_01_27.csv', index_col=0)\n",
    "file_name_list = list(df['File'].unique())\n",
    "train_split = int(len(file_name_list) * 0.9)\n",
    "print(train_split)\n",
    "Random(42).shuffle(file_name_list)\n",
    "train_files = file_name_list[:train_split]\n",
    "test_files = file_name_list[train_split:]\n",
    "X_train_text, y_train_text = load_data.load_relation_data_diabete(df, train_files, is_bert=True)\n",
    "X_test_text, y_test_text = load_data.load_relation_data_diabete(df, test_files, is_bert=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(analyzer = \"word\", binary = False, \n",
    "                             ngram_range=(3,3))\n",
    "# vectorizer = CountVectorizer(analyzer = \"word\", binary = False, \n",
    "#                              ngram_range=(3,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Len X train: 854\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SVC()"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f'Len X train: {len(X_train_text)}')\n",
    "X_train = vectorizer.fit_transform(X_train_text)\n",
    "y_train = [utils.relation2id[elem] for elem in y_train_text]\n",
    "clf = svm.SVC(decision_function_shape='ovr')\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Len X test: 117\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      causes       0.00      0.00      0.00         6\n",
      "    prevents       0.00      0.00      0.00         4\n",
      "      treats       0.71      0.71      0.71        14\n",
      "         has       0.85      1.00      0.92        86\n",
      "   diagnoses       1.00      0.29      0.44         7\n",
      " complicates       0.00      0.00      0.00         0\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       117\n",
      "   macro avg       0.43      0.33      0.35       117\n",
      "weighted avg       0.77      0.84      0.79       117\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lucas/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/lucas/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/lucas/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/lucas/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/lucas/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/lucas/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(f'Len X test: {len(X_test_text)}')\n",
    "X_test = vectorizer.transform(X_test_text)\n",
    "y_test = [utils.relation2id[elem] for elem in y_test_text]\n",
    "y_pred = clf.predict(X_test)\n",
    "labels = list(range(1, len(utils.RELATIONS)))\n",
    "target_names = utils.RELATIONS[1:]\n",
    "print(classification_report(y_test, y_pred, labels=labels, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create BERT RE data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sents</th>\n",
       "      <th>relations</th>\n",
       "      <th>relations_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>em relação a [E2]gelatina[/E2] , caso ela seja...</td>\n",
       "      <td>has(e1,e2)</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a [E1]glicada[/E1] é um exame chamado de \" hem...</td>\n",
       "      <td>diagnoses(e1,e2)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a glicada é um exame chamado de \" [E1]hemoglob...</td>\n",
       "      <td>diagnoses(e1,e2)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a glicada é um exame chamado de \" hemoglobina ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a [E2]glicada[/E2] é um exame chamado de \" hem...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1466</th>\n",
       "      <td>quem tem diabetes tem uma chance maior de ter ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1467</th>\n",
       "      <td>quem tem diabetes tem uma chance maior de ter ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1468</th>\n",
       "      <td>quem tem diabetes tem uma chance maior de ter ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1469</th>\n",
       "      <td>quem tem diabetes tem uma chance maior de ter ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1470</th>\n",
       "      <td>quem tem diabetes tem uma chance maior de ter ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1471 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  sents         relations  \\\n",
       "0     em relação a [E2]gelatina[/E2] , caso ela seja...        has(e1,e2)   \n",
       "1     a [E1]glicada[/E1] é um exame chamado de \" hem...  diagnoses(e1,e2)   \n",
       "2     a glicada é um exame chamado de \" [E1]hemoglob...  diagnoses(e1,e2)   \n",
       "3     a glicada é um exame chamado de \" hemoglobina ...             Other   \n",
       "4     a [E2]glicada[/E2] é um exame chamado de \" hem...             Other   \n",
       "...                                                 ...               ...   \n",
       "1466  quem tem diabetes tem uma chance maior de ter ...             Other   \n",
       "1467  quem tem diabetes tem uma chance maior de ter ...             Other   \n",
       "1468  quem tem diabetes tem uma chance maior de ter ...             Other   \n",
       "1469  quem tem diabetes tem uma chance maior de ter ...             Other   \n",
       "1470  quem tem diabetes tem uma chance maior de ter ...             Other   \n",
       "\n",
       "      relations_id  \n",
       "0                4  \n",
       "1                5  \n",
       "2                5  \n",
       "3                0  \n",
       "4                0  \n",
       "...            ...  \n",
       "1466             0  \n",
       "1467             0  \n",
       "1468             0  \n",
       "1469             0  \n",
       "1470             0  \n",
       "\n",
       "[1471 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = load_data.create_bert_re_df(X_train_text, y_train_text)\n",
    "df_test = load_data.create_bert_re_df(X_test_text, y_test_text)\n",
    "# df_train = df_train[:150]\n",
    "# df_test = df_test[:100]\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sents</th>\n",
       "      <th>relations</th>\n",
       "      <th>relations_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a pele dos pés de pessoas com [E2]diabetes[/E2...</td>\n",
       "      <td>prevents(e1,e2)</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a pele dos pés de pessoas com diabetes costuma...</td>\n",
       "      <td>prevents(e1,e2)</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a pele dos pés de pessoas com diabetes costuma...</td>\n",
       "      <td>prevents(e1,e2)</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a pele dos pés de pessoas com [E1]diabetes[/E1...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>é recomendado que se calibre o [E1]glicosímetr...</td>\n",
       "      <td>diagnoses(e1,e2)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>níveis - que é o seu caso - podem levar a uma ...</td>\n",
       "      <td>treats(e1,e2)</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>níveis - que é o seu caso - podem levar a uma ...</td>\n",
       "      <td>treats(e1,e2)</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>níveis - que é o seu caso - podem levar a uma ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>níveis - que é o seu caso - podem levar a uma ...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>[E2]níveis[/E2] - que é o seu caso - podem lev...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>196 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 sents         relations  \\\n",
       "0    a pele dos pés de pessoas com [E2]diabetes[/E2...   prevents(e1,e2)   \n",
       "1    a pele dos pés de pessoas com diabetes costuma...   prevents(e1,e2)   \n",
       "2    a pele dos pés de pessoas com diabetes costuma...   prevents(e1,e2)   \n",
       "3    a pele dos pés de pessoas com [E1]diabetes[/E1...             Other   \n",
       "4    é recomendado que se calibre o [E1]glicosímetr...  diagnoses(e1,e2)   \n",
       "..                                                 ...               ...   \n",
       "191  níveis - que é o seu caso - podem levar a uma ...     treats(e1,e2)   \n",
       "192  níveis - que é o seu caso - podem levar a uma ...     treats(e1,e2)   \n",
       "193  níveis - que é o seu caso - podem levar a uma ...             Other   \n",
       "194  níveis - que é o seu caso - podem levar a uma ...             Other   \n",
       "195  [E2]níveis[/E2] - que é o seu caso - podem lev...             Other   \n",
       "\n",
       "     relations_id  \n",
       "0               2  \n",
       "1               2  \n",
       "2               2  \n",
       "3               0  \n",
       "4               5  \n",
       "..            ...  \n",
       "191             3  \n",
       "192             3  \n",
       "193             0  \n",
       "194             0  \n",
       "195             0  \n",
       "\n",
       "[196 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(df_train, open('../../../BERT-Relation-Extraction/data/df_train_bete.pkl', 'wb'))\n",
    "pickle.dump(df_test, open('../../../BERT-Relation-Extraction/data/df_test_bete.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parse eHealth data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Len eHealth train: 20571\n",
      "Len eHealth test: 1658\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sents</th>\n",
       "      <th>relations</th>\n",
       "      <th>relations_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>La presencia del [E1]gen[/E1] de células falci...</td>\n",
       "      <td>in-context(e1,e2)</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>La [E1]presencia[/E1] del [E2]gen de células f...</td>\n",
       "      <td>subject(e1,e2)</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>La [E1]presencia[/E1] del gen de células falci...</td>\n",
       "      <td>same-as(e1,e2)</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>La [E1]presencia[/E1] del [E2]gen[/E2] de célu...</td>\n",
       "      <td>subject(e1,e2)</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>La [E1]presencia[/E1] del gen de células falci...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20566</th>\n",
       "      <td>Durante la [E1]noche[/E1] el edificio de Venez...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20567</th>\n",
       "      <td>Durante la [E1]noche[/E1] el edificio de Venez...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20568</th>\n",
       "      <td>Durante la [E1]noche[/E1] el edificio de Venez...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20569</th>\n",
       "      <td>Durante la [E1]noche[/E1] el edificio de Venez...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20570</th>\n",
       "      <td>Durante la noche el [E1]edificio[/E1] de Venez...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20571 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   sents          relations  \\\n",
       "0      La presencia del [E1]gen[/E1] de células falci...  in-context(e1,e2)   \n",
       "1      La [E1]presencia[/E1] del [E2]gen de células f...     subject(e1,e2)   \n",
       "2      La [E1]presencia[/E1] del gen de células falci...     same-as(e1,e2)   \n",
       "3      La [E1]presencia[/E1] del [E2]gen[/E2] de célu...     subject(e1,e2)   \n",
       "4      La [E1]presencia[/E1] del gen de células falci...              Other   \n",
       "...                                                  ...                ...   \n",
       "20566  Durante la [E1]noche[/E1] el edificio de Venez...              Other   \n",
       "20567  Durante la [E1]noche[/E1] el edificio de Venez...              Other   \n",
       "20568  Durante la [E1]noche[/E1] el edificio de Venez...              Other   \n",
       "20569  Durante la [E1]noche[/E1] el edificio de Venez...              Other   \n",
       "20570  Durante la noche el [E1]edificio[/E1] de Venez...              Other   \n",
       "\n",
       "       relations_id  \n",
       "0                 6  \n",
       "1                 9  \n",
       "2                13  \n",
       "3                 9  \n",
       "4                 0  \n",
       "...             ...  \n",
       "20566             0  \n",
       "20567             0  \n",
       "20568             0  \n",
       "20569             0  \n",
       "20570             0  \n",
       "\n",
       "[20571 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ehealth_df_train = load_data.load_relation_data_ehealth('../../data/eHealthKD/original/ref/training/input_multilingual.json')\n",
    "ehealth_df_test = load_data.load_relation_data_ehealth('../../data/eHealthKD/original/ref/develop/input_multilingual.json')\n",
    "print(f'Len eHealth train: {len(ehealth_df_train)}')\n",
    "print(f'Len eHealth test: {len(ehealth_df_test)}')\n",
    "ehealth_df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(ehealth_df_train, open('../../../BERT-Relation-Extraction/data/df_train_ehealth.pkl', 'wb'))\n",
    "pickle.dump(ehealth_df_test, open('../../../BERT-Relation-Extraction/data/df_test_ehealth.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "cf635ce910e2edf481a55afeb7dc9764fd36fcc0eb4136e934e045e245c56289"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 64-bit ('wecnlp': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}