03_same_weights.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, RandomSampler, Dataset\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed_value):\n",
    "    random.seed(seed_value)\n",
    "    np.random.seed(seed_value)\n",
    "    torch.manual_seed(seed_value)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed_value)\n",
    "\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_everything(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('./data/train_result_mean_median_max_min_current_next_prev.csv')\n",
    "df_test = pd.read_csv('./data/test_result_mean_median_max_min_current_next_prev.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>...</th>\n",
       "      <th>X2_next_min</th>\n",
       "      <th>X4_next_min</th>\n",
       "      <th>X2_prev_min</th>\n",
       "      <th>X4_prev_min</th>\n",
       "      <th>X2_current_min</th>\n",
       "      <th>X4_current_min</th>\n",
       "      <th>to_next_game_1</th>\n",
       "      <th>to_next_game_2</th>\n",
       "      <th>to_prev_game_1</th>\n",
       "      <th>to_prev_game_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>908749</td>\n",
       "      <td>954</td>\n",
       "      <td>480270</td>\n",
       "      <td>935</td>\n",
       "      <td>71</td>\n",
       "      <td>79</td>\n",
       "      <td>53</td>\n",
       "      <td>89</td>\n",
       "      <td>...</td>\n",
       "      <td>780</td>\n",
       "      <td>905</td>\n",
       "      <td>867</td>\n",
       "      <td>931</td>\n",
       "      <td>954</td>\n",
       "      <td>935</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>438444</td>\n",
       "      <td>2162</td>\n",
       "      <td>486685</td>\n",
       "      <td>2154</td>\n",
       "      <td>32</td>\n",
       "      <td>39</td>\n",
       "      <td>48</td>\n",
       "      <td>49</td>\n",
       "      <td>...</td>\n",
       "      <td>2110</td>\n",
       "      <td>2096</td>\n",
       "      <td>2224</td>\n",
       "      <td>2126</td>\n",
       "      <td>2026</td>\n",
       "      <td>2034</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>596915</td>\n",
       "      <td>2066</td>\n",
       "      <td>711059</td>\n",
       "      <td>2081</td>\n",
       "      <td>43</td>\n",
       "      <td>21</td>\n",
       "      <td>58</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>1822</td>\n",
       "      <td>2153</td>\n",
       "      <td>2092</td>\n",
       "      <td>2047</td>\n",
       "      <td>1883</td>\n",
       "      <td>2053</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>625198</td>\n",
       "      <td>1508</td>\n",
       "      <td>66810</td>\n",
       "      <td>1474</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>73</td>\n",
       "      <td>...</td>\n",
       "      <td>1502</td>\n",
       "      <td>1416</td>\n",
       "      <td>1506</td>\n",
       "      <td>1564</td>\n",
       "      <td>1502</td>\n",
       "      <td>1474</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>228654</td>\n",
       "      <td>1202</td>\n",
       "      <td>542816</td>\n",
       "      <td>1196</td>\n",
       "      <td>53</td>\n",
       "      <td>35</td>\n",
       "      <td>16</td>\n",
       "      <td>80</td>\n",
       "      <td>...</td>\n",
       "      <td>1263</td>\n",
       "      <td>1264</td>\n",
       "      <td>1239</td>\n",
       "      <td>1162</td>\n",
       "      <td>1202</td>\n",
       "      <td>1191</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 52 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  X0      X1    X2      X3    X4  X5  X6  X7  X8  ...  X2_next_min  \\\n",
       "0   0   2  908749   954  480270   935  71  79  53  89  ...          780   \n",
       "1   1   1  438444  2162  486685  2154  32  39  48  49  ...         2110   \n",
       "2   2   1  596915  2066  711059  2081  43  21  58  94  ...         1822   \n",
       "3   3   1  625198  1508   66810  1474   7   1  34  73  ...         1502   \n",
       "4   4   2  228654  1202  542816  1196  53  35  16  80  ...         1263   \n",
       "\n",
       "   X4_next_min  X2_prev_min  X4_prev_min  X2_current_min  X4_current_min  \\\n",
       "0          905          867          931             954             935   \n",
       "1         2096         2224         2126            2026            2034   \n",
       "2         2153         2092         2047            1883            2053   \n",
       "3         1416         1506         1564            1502            1474   \n",
       "4         1264         1239         1162            1202            1191   \n",
       "\n",
       "   to_next_game_1  to_next_game_2  to_prev_game_1  to_prev_game_2  \n",
       "0              40               1               2               7  \n",
       "1              17              13               8               1  \n",
       "2               1               1               2               4  \n",
       "3               1               1               1              26  \n",
       "4               8              10              19               1  \n",
       "\n",
       "[5 rows x 52 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "binarizer = MultiLabelBinarizer(classes=np.arange(0,101))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  \n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "fig_1_train = binarizer.fit_transform(df_train.iloc[:, 6:14].as_matrix())\n",
    "fig_2_train = binarizer.transform(df_train.iloc[:, 14:22].as_matrix())\n",
    "fig_1_test = binarizer.transform(df_test.iloc[:, 6:14].as_matrix())\n",
    "fig_2_test = binarizer.transform(df_test.iloc[:, 14:22].as_matrix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['1/to_next_game_1'] = df_train['to_next_game_1'].apply(lambda x: x if x==0 else 1/x)\n",
    "df_train['1/to_next_game_2'] = df_train['to_next_game_2'].apply(lambda x: x if x==0 else 1/x)\n",
    "df_train['1/to_prev_game_1'] = df_train['to_prev_game_1'].apply(lambda x: x if x==0 else 1/x)\n",
    "df_train['1/to_prev_game_2'] = df_train['to_prev_game_2'].apply(lambda x: x if x==0 else 1/x)\n",
    "\n",
    "df_test['1/to_next_game_1'] = df_test['to_next_game_1'].apply(lambda x: x if x==0 else 1/x)\n",
    "df_test['1/to_next_game_2'] = df_test['to_next_game_2'].apply(lambda x: x if x==0 else 1/x)\n",
    "df_test['1/to_prev_game_1'] = df_test['to_prev_game_1'].apply(lambda x: x if x==0 else 1/x)\n",
    "df_test['1/to_prev_game_2'] = df_test['to_prev_game_2'].apply(lambda x: x if x==0 else 1/x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>...</th>\n",
       "      <th>X2_current_min</th>\n",
       "      <th>X4_current_min</th>\n",
       "      <th>to_next_game_1</th>\n",
       "      <th>to_next_game_2</th>\n",
       "      <th>to_prev_game_1</th>\n",
       "      <th>to_prev_game_2</th>\n",
       "      <th>1/to_next_game_1</th>\n",
       "      <th>1/to_next_game_2</th>\n",
       "      <th>1/to_prev_game_1</th>\n",
       "      <th>1/to_prev_game_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>908749</td>\n",
       "      <td>954</td>\n",
       "      <td>480270</td>\n",
       "      <td>935</td>\n",
       "      <td>71</td>\n",
       "      <td>79</td>\n",
       "      <td>53</td>\n",
       "      <td>89</td>\n",
       "      <td>...</td>\n",
       "      <td>954</td>\n",
       "      <td>935</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>0.025000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>438444</td>\n",
       "      <td>2162</td>\n",
       "      <td>486685</td>\n",
       "      <td>2154</td>\n",
       "      <td>32</td>\n",
       "      <td>39</td>\n",
       "      <td>48</td>\n",
       "      <td>49</td>\n",
       "      <td>...</td>\n",
       "      <td>2026</td>\n",
       "      <td>2034</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>596915</td>\n",
       "      <td>2066</td>\n",
       "      <td>711059</td>\n",
       "      <td>2081</td>\n",
       "      <td>43</td>\n",
       "      <td>21</td>\n",
       "      <td>58</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>1883</td>\n",
       "      <td>2053</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>625198</td>\n",
       "      <td>1508</td>\n",
       "      <td>66810</td>\n",
       "      <td>1474</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>73</td>\n",
       "      <td>...</td>\n",
       "      <td>1502</td>\n",
       "      <td>1474</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>228654</td>\n",
       "      <td>1202</td>\n",
       "      <td>542816</td>\n",
       "      <td>1196</td>\n",
       "      <td>53</td>\n",
       "      <td>35</td>\n",
       "      <td>16</td>\n",
       "      <td>80</td>\n",
       "      <td>...</td>\n",
       "      <td>1202</td>\n",
       "      <td>1191</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.052632</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3633000</td>\n",
       "      <td>3633000</td>\n",
       "      <td>2</td>\n",
       "      <td>644871</td>\n",
       "      <td>1211</td>\n",
       "      <td>421763</td>\n",
       "      <td>1194</td>\n",
       "      <td>24</td>\n",
       "      <td>65</td>\n",
       "      <td>54</td>\n",
       "      <td>86</td>\n",
       "      <td>...</td>\n",
       "      <td>1211</td>\n",
       "      <td>1049</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>166</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.006024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3633001</td>\n",
       "      <td>3633001</td>\n",
       "      <td>1</td>\n",
       "      <td>689210</td>\n",
       "      <td>699</td>\n",
       "      <td>569184</td>\n",
       "      <td>706</td>\n",
       "      <td>26</td>\n",
       "      <td>50</td>\n",
       "      <td>81</td>\n",
       "      <td>33</td>\n",
       "      <td>...</td>\n",
       "      <td>699</td>\n",
       "      <td>706</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.055556</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3633002</td>\n",
       "      <td>3633002</td>\n",
       "      <td>2</td>\n",
       "      <td>227914</td>\n",
       "      <td>1063</td>\n",
       "      <td>174332</td>\n",
       "      <td>1062</td>\n",
       "      <td>48</td>\n",
       "      <td>64</td>\n",
       "      <td>55</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1063</td>\n",
       "      <td>1036</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>257</td>\n",
       "      <td>0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003891</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3633003</td>\n",
       "      <td>3633003</td>\n",
       "      <td>2</td>\n",
       "      <td>1066558</td>\n",
       "      <td>1752</td>\n",
       "      <td>358523</td>\n",
       "      <td>1780</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>47</td>\n",
       "      <td>68</td>\n",
       "      <td>...</td>\n",
       "      <td>1752</td>\n",
       "      <td>1719</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3633004</td>\n",
       "      <td>3633004</td>\n",
       "      <td>2</td>\n",
       "      <td>525076</td>\n",
       "      <td>1981</td>\n",
       "      <td>693443</td>\n",
       "      <td>1974</td>\n",
       "      <td>80</td>\n",
       "      <td>43</td>\n",
       "      <td>7</td>\n",
       "      <td>37</td>\n",
       "      <td>...</td>\n",
       "      <td>1908</td>\n",
       "      <td>1970</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3633005 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  X0       X1    X2      X3    X4  X5  X6  X7  X8  ...  \\\n",
       "0              0   2   908749   954  480270   935  71  79  53  89  ...   \n",
       "1              1   1   438444  2162  486685  2154  32  39  48  49  ...   \n",
       "2              2   1   596915  2066  711059  2081  43  21  58  94  ...   \n",
       "3              3   1   625198  1508   66810  1474   7   1  34  73  ...   \n",
       "4              4   2   228654  1202  542816  1196  53  35  16  80  ...   \n",
       "...          ...  ..      ...   ...     ...   ...  ..  ..  ..  ..  ...   \n",
       "3633000  3633000   2   644871  1211  421763  1194  24  65  54  86  ...   \n",
       "3633001  3633001   1   689210   699  569184   706  26  50  81  33  ...   \n",
       "3633002  3633002   2   227914  1063  174332  1062  48  64  55   3  ...   \n",
       "3633003  3633003   2  1066558  1752  358523  1780  10  31  47  68  ...   \n",
       "3633004  3633004   2   525076  1981  693443  1974  80  43   7  37  ...   \n",
       "\n",
       "         X2_current_min  X4_current_min  to_next_game_1  to_next_game_2  \\\n",
       "0                   954             935              40               1   \n",
       "1                  2026            2034              17              13   \n",
       "2                  1883            2053               1               1   \n",
       "3                  1502            1474               1               1   \n",
       "4                  1202            1191               8              10   \n",
       "...                 ...             ...             ...             ...   \n",
       "3633000            1211            1049               6               0   \n",
       "3633001             699             706               0              18   \n",
       "3633002            1063            1036               5               1   \n",
       "3633003            1752            1719               0               1   \n",
       "3633004            1908            1970               1               1   \n",
       "\n",
       "         to_prev_game_1  to_prev_game_2  1/to_next_game_1  1/to_next_game_2  \\\n",
       "0                     2               7          0.025000          1.000000   \n",
       "1                     8               1          0.058824          0.076923   \n",
       "2                     2               4          1.000000          1.000000   \n",
       "3                     1              26          1.000000          1.000000   \n",
       "4                    19               1          0.125000          0.100000   \n",
       "...                 ...             ...               ...               ...   \n",
       "3633000              16             166          0.166667          0.000000   \n",
       "3633001               4              10          0.000000          0.055556   \n",
       "3633002             257               0          0.200000          1.000000   \n",
       "3633003               4               1          0.000000          1.000000   \n",
       "3633004               7               4          1.000000          1.000000   \n",
       "\n",
       "         1/to_prev_game_1  1/to_prev_game_2  \n",
       "0                0.500000          0.142857  \n",
       "1                0.125000          1.000000  \n",
       "2                0.500000          0.250000  \n",
       "3                1.000000          0.038462  \n",
       "4                0.052632          1.000000  \n",
       "...                   ...               ...  \n",
       "3633000          0.062500          0.006024  \n",
       "3633001          0.250000          0.100000  \n",
       "3633002          0.003891          0.000000  \n",
       "3633003          0.250000          1.000000  \n",
       "3633004          0.142857          0.250000  \n",
       "\n",
       "[3633005 rows x 56 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "rating_1_train = df_train[['X2',\n",
    "                           'X2_next_mean',  'X2_prev_mean','X2_current_mean',\n",
    "                           'X2_next_median', 'X2_prev_median','X2_current_median',\n",
    "                           'X2_next_max', 'X2_prev_max', 'X2_current_max', \n",
    "                           'X2_next_min', 'X2_prev_min', 'X2_current_min',\n",
    "                           'to_next_game_1', 'to_prev_game_1', \n",
    "                           '1/to_next_game_1','1/to_prev_game_1']].values\n",
    "rating_2_train = df_train[['X4',\n",
    "                           'X4_next_mean',  'X4_prev_mean','X4_current_mean',\n",
    "                           'X4_next_median', 'X4_prev_median','X4_current_median',\n",
    "                           'X4_next_max', 'X4_prev_max', 'X4_current_max', \n",
    "                           'X4_next_min', 'X4_prev_min', 'X4_current_min',\n",
    "                           'to_next_game_2', 'to_prev_game_2', \n",
    "                           '1/to_next_game_2','1/to_prev_game_2']].values\n",
    "\n",
    "rating_1_test =  df_test[['X2',\n",
    "                           'X2_next_mean',  'X2_prev_mean','X2_current_mean',\n",
    "                           'X2_next_median', 'X2_prev_median','X2_current_median',\n",
    "                           'X2_next_max', 'X2_prev_max', 'X2_current_max', \n",
    "                           'X2_next_min', 'X2_prev_min', 'X2_current_min',\n",
    "                           'to_next_game_1', 'to_prev_game_1', \n",
    "                           '1/to_next_game_1','1/to_prev_game_1']].values\n",
    "rating_2_test = df_test[['X4',\n",
    "                           'X4_next_mean',  'X4_prev_mean','X4_current_mean',\n",
    "                           'X4_next_median', 'X4_prev_median','X4_current_median',\n",
    "                           'X4_next_max', 'X4_prev_max', 'X4_current_max', \n",
    "                           'X4_next_min', 'X4_prev_min', 'X4_current_min',\n",
    "                           'to_next_game_2', 'to_prev_game_2', \n",
    "                           '1/to_next_game_2','1/to_prev_game_2']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = df_train['target']\n",
    "rating_1_train, rating_2_train, fig_1_train, fig_2_train, y_train = shuffle(rating_1_train, rating_2_train, fig_1_train, fig_2_train, y_train, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "rating_1_train, rating_1_val, \\\n",
    "rating_2_train, rating_2_val, \\\n",
    "fig_1_train, fig_1_val, \\\n",
    "fig_2_train, fig_2_val, \\\n",
    "y_train, y_val = train_test_split(rating_1_train, rating_2_train, fig_1_train, fig_2_train, y_train, random_state=42, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# figs_train = np.concatenate([fig_1_train, fig_2_train], axis=1)\n",
    "# figs_val = np.concatenate([fig_1_val, fig_2_val], axis=1)\n",
    "# figs_test = np.concatenate([fig_1_test, fig_2_test], axis=1)\n",
    "\n",
    "# rating_train = np.concatenate([rating_1_train, rating_2_train], axis=1)\n",
    "# rating_val = np.concatenate([rating_1_val, rating_2_val], axis=1)\n",
    "# rating_test = np.concatenate([rating_1_test, rating_2_test], axis=1)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data for torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainDataset(Dataset):\n",
    "    def __init__(self, figs_1, figs_2, rating_1, rating_2, y):\n",
    "        self.figs_1 = torch.tensor(figs_1, dtype=torch.float)\n",
    "        self.figs_2 = torch.tensor(figs_2, dtype=torch.float)\n",
    "        self.rating_1 = torch.tensor(rating_1, dtype=torch.float)\n",
    "        self.rating_2 = torch.tensor(rating_2, dtype=torch.float)\n",
    "        self.y = torch.tensor(y.tolist(), dtype=torch.float)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.figs_1)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        # можно добавить случайный своп\n",
    "        return {\n",
    "            'figs_1': self.figs_1[idx],\n",
    "            'figs_2': self.figs_2[idx],\n",
    "            'rating_1': self.rating_1[idx],\n",
    "            'rating_2': self.rating_2[idx],\n",
    "            'y': self.y[idx]\n",
    "        }\n",
    "\n",
    "class TestDataset(Dataset):\n",
    "    def __init__(self,  figs_1, figs_2, rating_1, rating_2):\n",
    "        self.figs_1 = torch.tensor(figs_1, dtype=torch.float)\n",
    "        self.figs_2 = torch.tensor(figs_2, dtype=torch.float)\n",
    "        self.rating_1 = torch.tensor(rating_1, dtype=torch.float)\n",
    "        self.rating_2 = torch.tensor(rating_2, dtype=torch.float)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.figs_1)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return {\n",
    "            'figs_1': self.figs_1[idx],\n",
    "            'figs_2': self.figs_2[idx],\n",
    "            'rating_1': self.rating_1[idx],\n",
    "            'rating_2': self.rating_2[idx],\n",
    "        }\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(iterator, optimizer, criterion):\n",
    "        model.train()\n",
    "        epoch_loss = 0\n",
    "\n",
    "        for i, batch in tqdm(enumerate(iterator)):\n",
    "            targets = batch['y'].to(device)\n",
    "            figs_1 = batch['figs_1'].to(device)\n",
    "            figs_2 = batch['figs_2'].to(device)\n",
    "            rating_1 = batch['rating_1'].to(device)\n",
    "            rating_2 = batch['rating_2'].to(device)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            logits = model(figs_1, figs_2, rating_1, rating_2).squeeze()\n",
    "            loss = criterion(logits, targets)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "\n",
    "        return epoch_loss / len(iterator)\n",
    "    \n",
    "def val_epoch(iterator,criterion):\n",
    "        model.eval()\n",
    "        epoch_loss = 0\n",
    "        with torch.no_grad():\n",
    "            for i, batch in tqdm(enumerate(iterator)):\n",
    "                targets = batch['y'].to(device)\n",
    "                figs_1 = batch['figs_1'].to(device)\n",
    "                figs_2 = batch['figs_2'].to(device)\n",
    "                rating_1 = batch['rating_1'].to(device)\n",
    "                rating_2 = batch['rating_2'].to(device)\n",
    "    \n",
    "                logits = model(figs_1, figs_2, rating_1, rating_2).squeeze()\n",
    "                loss = criterion(logits, targets)\n",
    "                epoch_loss += loss.item()\n",
    "\n",
    "        return epoch_loss / len(iterator)\n",
    "    \n",
    "def predict(iterator):\n",
    "        model.eval()\n",
    "        preds = []\n",
    "        with torch.no_grad():\n",
    "            for i, batch in tqdm(enumerate(iterator)):\n",
    "                figs_1 = batch['figs_1'].to(device)\n",
    "                figs_2 = batch['figs_2'].to(device)\n",
    "                rating_1 = batch['rating_1'].to(device)\n",
    "                rating_2 = batch['rating_2'].to(device)\n",
    "                pred = model.predict(figs_1, figs_2, rating_1, rating_2)\n",
    "                preds += pred.to('cpu').numpy().tolist()\n",
    "\n",
    "        return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(NN, self).__init__()\n",
    "        self.fc1figs = nn.Linear(101, 128)\n",
    "        self.bn1figs = nn.BatchNorm1d(128)        \n",
    "        self.fc2figs = nn.Linear(128, 32)\n",
    "        self.bn2figs = nn.BatchNorm1d(32)\n",
    "       \n",
    "        self.fc1rating = nn.Linear(17, 32)\n",
    "        self.bn1rating = nn.BatchNorm1d(32)\n",
    "        self.fc2rating = nn.Linear(32, 16)\n",
    "        self.bn2rating = nn.BatchNorm1d(16)\n",
    "        \n",
    "        \n",
    "        \n",
    "        self.fc1res = nn.Linear(32*1 + 16*1, 32)\n",
    "        self.bn1res = nn.BatchNorm1d(32)\n",
    "        self.fc2res = nn.Linear(32, 1)\n",
    "        \n",
    "        self.do = nn.Dropout(0.2)\n",
    "    \n",
    "    def predict(self, figs_1, figs_2, rating_1, rating_2):\n",
    "        result = self.forward(figs_1, figs_2, rating_1, rating_2)\n",
    "        result = F.sigmoid(result)\n",
    "        return result\n",
    "        \n",
    "    def forward(self, figs_1, figs_2, rating_1, rating_2):\n",
    "        figs_1 = self.fc1figs(figs_1)\n",
    "        figs_1 = self.bn1figs(figs_1)\n",
    "        figs_1 = F.relu(figs_1)\n",
    "        #figs_1 = self.do(figs_1)\n",
    "        \n",
    "        figs_1 = self.fc2figs(figs_1)\n",
    "        figs_1 = self.bn2figs(figs_1)\n",
    "        figs_1 = F.relu(figs_1)\n",
    "        #figs_1 = self.do(figs_1)\n",
    "        \n",
    "        figs_2 = self.fc1figs(figs_2)\n",
    "        figs_2 = self.bn1figs(figs_2)\n",
    "        figs_2 = F.relu(figs_2)\n",
    "        #figs_2 = self.do(figs_2)\n",
    "        \n",
    "        figs_2 = self.fc2figs(figs_2)\n",
    "        figs_2 = self.bn2figs(figs_2)\n",
    "        figs_2 = F.relu(figs_2)\n",
    "        #figs_2 = self.do(figs_2)\n",
    "        \n",
    "        figs_diff = figs_1 - figs_2\n",
    "        #figs_cross = torch.cross(figs_1, figs_2)\n",
    "        \n",
    "        \n",
    "        rating_1 = self.fc1rating(rating_1)\n",
    "        rating_1 = self.bn1rating(rating_1)\n",
    "        rating_1 = F.relu(rating_1)\n",
    "        #rating_1 = self.do(rating_1)\n",
    "        \n",
    "        rating_1 = self.fc2rating(rating_1)\n",
    "        rating_1 = self.bn2rating(rating_1)\n",
    "        rating_1 = F.relu(rating_1)\n",
    "        #rating_1 = self.do(rating_1)\n",
    "        \n",
    "        \n",
    "        rating_2 = self.fc1rating(rating_2)\n",
    "        rating_2 = self.bn1rating(rating_2)\n",
    "        rating_2 = F.relu(rating_2)\n",
    "        #rating_2 = self.do(rating_2)\n",
    "        \n",
    "        rating_2 = self.fc2rating(rating_2)\n",
    "        rating_2 = self.bn2rating(rating_2)\n",
    "        rating_2 = F.relu(rating_2)\n",
    "        #rating_2 = self.do(rating_2)\n",
    "        \n",
    "        rating_diff = rating_1 - rating_2\n",
    "        #rating_cross = torch.cross(rating_1, rating_2)\n",
    "        \n",
    "        features = torch.cat((figs_diff, rating_diff), dim=-1)\n",
    "        features = self.fc1res(features)\n",
    "        features = self.bn1res(features)\n",
    "        features = F.relu(features)\n",
    "        #features = self.do(features)\n",
    "        \n",
    "        result = self.fc2res(features)\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "train_dataset = TrainDataset(fig_1_train, fig_2_train, rating_1_train, rating_2_train, y_train)\n",
    "train_sampler = RandomSampler(train_dataset)\n",
    "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size)\n",
    "\n",
    "valid_dataset = TrainDataset(fig_1_val, fig_2_val, rating_1_val, rating_2_val, y_val)\n",
    "valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size)\n",
    "\n",
    "test_dataset = TestDataset(fig_1_test, fig_2_test, rating_1_test, rating_2_test)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2906404, 17)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_2_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NN()\n",
    "if device == 'cuda':\n",
    "    model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 40\n",
    "criterion = torch.nn.BCEWithLogitsLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters())\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.7, last_epoch=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:48, 157.59it/s]\n",
      "11354it [00:20, 549.69it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 \n",
      " Train Loss 0.5119795  Val loss 0.5027951:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:48, 157.20it/s]\n",
      "11354it [00:20, 554.42it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 \n",
      " Train Loss 0.4964456  Val loss 0.4888294:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:49, 157.13it/s]\n",
      "11354it [00:20, 561.13it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 \n",
      " Train Loss 0.49204  Val loss 0.5031868:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:48, 157.50it/s]\n",
      "11354it [00:20, 559.89it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 \n",
      " Train Loss 0.4895085  Val loss 0.4938149:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:49, 157.12it/s]\n",
      "11354it [00:20, 564.22it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 \n",
      " Train Loss 0.4875982  Val loss 0.4939599:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:48, 157.25it/s]\n",
      "11354it [00:20, 561.97it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 \n",
      " Train Loss 0.4841822  Val loss 0.4794521:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:47, 157.88it/s]\n",
      "11354it [00:20, 560.11it/s]\n",
      "1it [00:00,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 \n",
      " Train Loss 0.4832627  Val loss 0.4892297:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:48, 157.20it/s]\n",
      "11354it [00:20, 549.46it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 \n",
      " Train Loss 0.4831575  Val loss 0.4792846:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:49, 156.81it/s]\n",
      "11354it [00:20, 558.72it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 \n",
      " Train Loss 0.4822065  Val loss 0.4768918:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:51, 155.84it/s]\n",
      "11354it [00:20, 553.54it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 \n",
      " Train Loss 0.4818058  Val loss 0.4809223:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:50, 156.27it/s]\n",
      "11354it [00:20, 558.29it/s]\n",
      "1it [00:00,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 \n",
      " Train Loss 0.4798476  Val loss 0.4774952:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:50, 156.52it/s]\n",
      "11354it [00:20, 551.23it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 \n",
      " Train Loss 0.4793388  Val loss 0.4804662:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:53, 154.58it/s]\n",
      "11354it [00:20, 548.04it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 \n",
      " Train Loss 0.4789578  Val loss 0.4819056:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:53, 154.56it/s]\n",
      "11354it [00:20, 546.04it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 \n",
      " Train Loss 0.4786849  Val loss 0.4800325:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:52, 155.35it/s]\n",
      "11354it [00:20, 553.13it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 \n",
      " Train Loss 0.4785399  Val loss 0.4761355:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:52, 155.10it/s]\n",
      "11354it [00:20, 545.10it/s]\n",
      "1it [00:00,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 \n",
      " Train Loss 0.4772552  Val loss 0.4756961:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:51, 155.68it/s]\n",
      "11354it [00:20, 552.23it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 \n",
      " Train Loss 0.4769802  Val loss 0.4725997:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:51, 155.83it/s]\n",
      "11354it [00:20, 550.67it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 \n",
      " Train Loss 0.4768484  Val loss 0.4737153:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:52, 155.22it/s]\n",
      "11354it [00:20, 554.77it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 \n",
      " Train Loss 0.4766726  Val loss 0.471223:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:51, 155.58it/s]\n",
      "11354it [00:20, 544.80it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 \n",
      " Train Loss 0.4764469  Val loss 0.4709992:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:53, 154.67it/s]\n",
      "11354it [00:20, 557.92it/s]\n",
      "1it [00:00,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 \n",
      " Train Loss 0.475492  Val loss 0.4710876:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:50, 156.12it/s]\n",
      "11354it [00:20, 562.84it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 \n",
      " Train Loss 0.4754835  Val loss 0.4722112:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:48, 157.61it/s]\n",
      "11354it [00:20, 566.29it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 \n",
      " Train Loss 0.4752268  Val loss 0.4712476:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [04:48, 157.15it/s]\n",
      "11354it [00:21, 539.05it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 \n",
      " Train Loss 0.475178  Val loss 0.4711985:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:03, 149.46it/s]\n",
      "11354it [00:21, 524.01it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 \n",
      " Train Loss 0.475099  Val loss 0.4708315:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:09, 146.87it/s]\n",
      "11354it [00:21, 522.38it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 \n",
      " Train Loss 0.4744337  Val loss 0.4713792:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:11, 145.70it/s]\n",
      "11354it [00:21, 524.68it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 \n",
      " Train Loss 0.4743461  Val loss 0.4718356:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:10, 146.44it/s]\n",
      "11354it [00:21, 520.74it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 \n",
      " Train Loss 0.4741577  Val loss 0.4699532:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:10, 146.39it/s]\n",
      "11354it [00:21, 527.32it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 \n",
      " Train Loss 0.47421  Val loss 0.4709034:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:12, 145.32it/s]\n",
      "11354it [00:21, 533.33it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 \n",
      " Train Loss 0.474098  Val loss 0.4707688:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:12, 145.52it/s]\n",
      "11354it [00:21, 518.22it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 \n",
      " Train Loss 0.4735872  Val loss 0.4705043:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:11, 145.91it/s]\n",
      "11354it [00:21, 526.64it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 \n",
      " Train Loss 0.4735718  Val loss 0.4700146:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:13, 144.81it/s]\n",
      "11354it [00:21, 521.52it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 \n",
      " Train Loss 0.4735159  Val loss 0.470532:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:10, 146.37it/s]\n",
      "11354it [00:21, 525.61it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 \n",
      " Train Loss 0.4734715  Val loss 0.4705668:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:07, 147.66it/s]\n",
      "11354it [00:21, 532.61it/s]\n",
      "1it [00:00,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 \n",
      " Train Loss 0.4733769  Val loss 0.4719339:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:10, 146.48it/s]\n",
      "11354it [00:21, 530.02it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 \n",
      " Train Loss 0.473006  Val loss 0.4710487:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:09, 146.66it/s]\n",
      "11354it [00:21, 519.79it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 \n",
      " Train Loss 0.4729356  Val loss 0.4694222:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:09, 146.56it/s]\n",
      "11354it [00:21, 524.07it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 \n",
      " Train Loss 0.4729987  Val loss 0.4710159:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:14, 144.59it/s]\n",
      "11354it [00:21, 522.67it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 \n",
      " Train Loss 0.4729019  Val loss 0.4709228:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45413it [05:14, 144.45it/s]\n",
      "11354it [00:22, 500.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 \n",
      " Train Loss 0.4729063  Val loss 0.4702604:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "best_valid_loss = float('inf')\n",
    "for epoch in range(1, epochs + 1):\n",
    "    train_loss = round(train_epoch(train_dataloader, optimizer, criterion), 7)\n",
    "    valid_loss = round(val_epoch(valid_dataloader, criterion), 7)\n",
    "    if valid_loss < best_valid_loss:\n",
    "            best_valid_loss = valid_loss\n",
    "            torch.save(model.state_dict(), './data/models/result_same_weights_no_dropout.pt')\n",
    "\n",
    "    print('Epoch: {} \\n Train Loss {}  Val loss {}:'.format(epoch, train_loss, valid_loss))\n",
    "    scheduler.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('./data/models/result_same_weights_no_dropout.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11354it [00:21, 517.75it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4694222"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(val_epoch(valid_dataloader, criterion), 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]/home/mpopov/.local/lib/python3.6/site-packages/torch/nn/functional.py:1351: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n",
      "14192it [00:24, 567.73it/s]\n"
     ]
    }
   ],
   "source": [
    "res = predict(test_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = [item[0] for item in res]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['target'] = res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test[['id','target']].to_csv('result_same_weights_no_dropout.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id,target\n",
      "0,0.44777318835258484\n",
      "1,0.01042748149484396\n",
      "2,0.4118897020816803\n",
      "3,0.15768353641033173\n",
      "4,0.19546987116336823\n",
      "5,0.15544505417346954\n",
      "6,0.7730392217636108\n",
      "7,0.3866017162799835\n",
      "8,0.8142403960227966\n"
     ]
    }
   ],
   "source": [
    "!head  result_same_weights_no_dropout.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}