Transfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#coding=utf-8\n",
    "import os\n",
    "import re\n",
    "import sys\n",
    "import yaml\n",
    "import argparse\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "# utils\n",
    "from utils import get_embedding, load_embed, save_embed, data_preprocessing\n",
    "# data\n",
    "from data import myDS, mytestDS\n",
    "# model\n",
    "from model import Siamese_lstm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/liushijing/anaconda3/bin/python\r\n"
     ]
    }
   ],
   "source": [
    "!which python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    'experiment_name': 'siamese-transfer-baseline',\n",
    "    'task': 'train',\n",
    "    'make_dict': True,\n",
    "    'data_preprocessing': True,\n",
    "\n",
    "    'ckpt_dir': 'ckpt/',\n",
    "\n",
    "    'training':{\n",
    "        'num_epochs': 20,\n",
    "        'learning_rate': 0.01,\n",
    "        'optimizer': 'sgd'\n",
    "    },\n",
    "    \n",
    "    'embedding':{\n",
    "        'full_embedding_path': 'input/wiki.es.vec',\n",
    "        'cur_embedding_path': 'input/embedding.pkl',\n",
    "    },\n",
    "        \n",
    "    'model':{\n",
    "        'fc_dropout': 0.1,\n",
    "        'fc_dim': 100,\n",
    "        'name': 'siamese',\n",
    "        'embed_size': 300,\n",
    "        'batch_size': 1,\n",
    "        'embedding_freeze': False,\n",
    "        'encoder':{\n",
    "            'hidden_size': 150,\n",
    "            'num_layers': 1,\n",
    "            'bidirectional': False,\n",
    "            'dropout': 0.5,\n",
    "        },  \n",
    "    },   \n",
    "    \n",
    "    'result':{\n",
    "        'filename':'result.txt',\n",
    "        'filepath':'res/',\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training data\n",
    "df_train_en_sp = pd.read_csv('./input/cikm_english_train_20180516.txt', sep='\t', header=None,\n",
    "                             error_bad_lines=False)\n",
    "df_train_sp_en = pd.read_csv('./input/cikm_spanish_train_20180516.txt', sep='\t', header=None,\n",
    "                             error_bad_lines=False)\n",
    "df_train_en_sp.columns = ['english1', 'spanish1', 'english2', 'spanish2', 'result']\n",
    "df_train_sp_en.columns = ['spanish1', 'english1', 'spanish2', 'english2', 'result']\n",
    "\n",
    "sp1 = pd.DataFrame(pd.concat([df_train_en_sp['spanish1'], df_train_sp_en['spanish1']], axis=0))\n",
    "sp2 = pd.DataFrame(pd.concat([df_train_en_sp['spanish2'], df_train_sp_en['spanish2']], axis=0))\n",
    "sp = pd.concat([sp1, sp2], axis=1).reset_index()\n",
    "sp = sp.drop(['index'], axis=1)\n",
    "\n",
    "en1 = pd.DataFrame(pd.concat([df_train_en_sp['english1'], df_train_sp_en['english1']], axis=0))\n",
    "en2 = pd.DataFrame(pd.concat([df_train_en_sp['english2'], df_train_sp_en['english2']], axis=0))\n",
    "en = pd.concat([en1, en2], axis=1).reset_index()\n",
    "en = en.drop(['index'], axis=1)\n",
    "\n",
    "result = pd.DataFrame(pd.concat([df_train_en_sp['result'], df_train_sp_en['result']], axis=0)).reset_index()\n",
    "result = result.drop(['index'], axis=1)\n",
    "# pd.get_dummies(result['result']).head()\n",
    "sp['label'] = result\n",
    "en['label'] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluation data\n",
    "test_data = pd.read_csv('./input/cikm_test_a_20180516.txt', sep='\t', header=None, error_bad_lines=False)\n",
    "test_data.columns = ['spanish1', 'spanish2']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean Up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_sent(sent):\n",
    "    sent = sent.lower()\n",
    "    sent = re.sub(u\"[_'\\-;%()|+&=*%.,!?:#$@\\[\\]/]\",' ',sent)\n",
    "    sent = re.sub('¡',' ',sent)\n",
    "    sent = re.sub('¿',' ',sent)\n",
    "    sent = re.sub('Á','á',sent)\n",
    "    sent = re.sub('Ó','ó',sent)\n",
    "    sent = re.sub('Ú','ú',sent)\n",
    "    sent = re.sub('É','é',sent)\n",
    "    sent = re.sub('Í','í',sent)\n",
    "    return sent\n",
    "def cleanSpanish(df):\n",
    "    if (sys.version_info > (3, 0)):\n",
    "        df['spanish1'] = df.spanish1.map(lambda x: ' '.join([word for word in\n",
    "                                                             nltk.word_tokenize(clean_sent(x))]))\n",
    "        df['spanish2'] = df.spanish2.map(lambda x: ' '.join([word for word in\n",
    "                                                             nltk.word_tokenize(clean_sent(x))]))\n",
    "    else:\n",
    "        df['spanish1'] = df.spanish1.map(lambda x: ' '.join([ word for word in\n",
    "                                                             nltk.word_tokenize(clean_sent(x).decode('utf-8'))]).encode('utf-8'))\n",
    "        df['spanish2'] = df.spanish2.map(lambda x: ' '.join([ word for word in\n",
    "                                                             nltk.word_tokenize(clean_sent(x).decode('utf-8'))]).encode('utf-8'))\n",
    "def removeSpanishStopWords(df, stop):\n",
    "    if (sys.version_info > (3, 0)):\n",
    "        df['spanish1'] = df.spanish1.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x)\n",
    "                                                             if word not in stop]))\n",
    "        df['spanish2'] = df.spanish2.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x)\n",
    "                                                             if word not in stop]))\n",
    "    else:\n",
    "        df['spanish1'] = df.spanish1.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x.decode('utf-8'))\n",
    "                                                             if word not in stop]).encode('utf-8'))\n",
    "        df['spanish2'] = df.spanish2.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x.decode('utf-8'))\n",
    "                                                             if word not in stop]).encode('utf-8'))\n",
    "        \n",
    "def cleanEnglish(df):\n",
    "    df['english1'] = df.english1.map(lambda x: ' '.join([word for word in\n",
    "                                                     nltk.word_tokenize(clean_sent(x))]))\n",
    "    df['english2'] = df.english2.map(lambda x: ' '.join([word for word in\n",
    "                                                     nltk.word_tokenize(clean_sent(x))]))\n",
    "\n",
    "def removeEnglishStopWords(df, stop):\n",
    "    df['english1'] = df.english1.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x) \n",
    "                                                               if word not in stop]))\n",
    "    df['english2'] = df.english2.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x) \n",
    "                                                               if word not in stop]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "sp_stops = set(stopwords.words(\"spanish\"))\n",
    "en_stops = set(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleanSpanish(sp)\n",
    "removeSpanishStopWords(sp, sp_stops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleanEnglish(en)\n",
    "removeEnglishStopWords(en, en_stops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleanSpanish(test_data)\n",
    "removeSpanishStopWords(test_data, sp_stops)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Empty Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_dirty_train(df):\n",
    "    df.replace('', np.nan, inplace=True)\n",
    "    dirty_data = df[df.isnull().any(axis=1)]\n",
    "    print('dirty sample count:', dirty_data.shape[0])\n",
    "    print('positive dirty training sample:', len(dirty_data[dirty_data['label'] == 1]))\n",
    "    print('negative dirty training sample:', len(dirty_data[dirty_data['label'] == 0]))\n",
    "    print('Before Clean:', len(df))\n",
    "    df = df.dropna()\n",
    "    print('After Clean:', len(df))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dirty sample count: 73\n",
      "positive dirty training sample: 5\n",
      "negative dirty training sample: 68\n",
      "Before Clean: 21400\n",
      "After Clean: 21327\n"
     ]
    }
   ],
   "source": [
    "sp = check_dirty_train(sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dirty sample count: 192\n",
      "positive dirty training sample: 20\n",
      "negative dirty training sample: 172\n",
      "Before Clean: 21400\n",
      "After Clean: 21208\n"
     ]
    }
   ],
   "source": [
    "en = check_dirty_train(en)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English sample count: 21208 \n",
      "Spanish sample count: 21327 \n",
      "Test sample count: 5000\n"
     ]
    }
   ],
   "source": [
    "test_data.replace('', np.nan, inplace=True)\n",
    "# refill the empty row\n",
    "test_data.iloc[1712, 0] = 'hola'\n",
    "test_data.iloc[2349, 0] = 'hola'\n",
    "test_data = test_data.dropna()\n",
    "print('English sample count:', en.shape[0], '\\nSpanish sample count:', sp.shape[0], '\\nTest sample count:', test_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp.to_csv(\"input/cleaned_sp.csv\", index=False)\n",
    "en.to_csv(\"input/cleaned_en.csv\", index=False)\n",
    "test_data.to_csv(\"input/cleaned_test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making embedding...\n",
      "Found 2606/2685 words with embedding vectors\n",
      "Missing Ratio: 2.94%\n",
      "Filled missing words' embeddings.\n",
      "Embedding Matrix Size:  2685\n",
      "Embedding saved\n",
      "Saved generated en embedding.\n"
     ]
    }
   ],
   "source": [
    "full_en_embed_path = 'input/wiki.en.vec'\n",
    "cur_en_embed_path = 'input/en_embed.pkl'\n",
    "\n",
    "print('Making embedding...')\n",
    "en_embed_dict = get_embedding(enDS.vocab._id2word, full_en_embed_path)\n",
    "save_embed(en_embed_dict, cur_en_embed_path)\n",
    "print('Saved generated en embedding.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making embedding...\n",
      "Found 5142/5767 words with embedding vectors\n",
      "Missing Ratio: 10.84%\n",
      "Filled missing words' embeddings.\n",
      "Embedding Matrix Size:  5767\n",
      "Embedding saved\n",
      "Saved generated sp embedding.\n"
     ]
    }
   ],
   "source": [
    "full_sp_embed_path = 'input/wiki.es.vec'\n",
    "cur_sp_embed_path = 'input/sp_embed.pkl'\n",
    "\n",
    "print('Making embedding...')\n",
    "sp_embed_dict = get_embedding(spDS.vocab._id2word, full_sp_embed_path)\n",
    "save_embed(sp_embed_dict, cur_sp_embed_path)\n",
    "print('Saved generated sp embedding.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sencond Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "en = pd.read_csv(\"input/cleaned_en.csv\")\n",
    "sp = pd.read_csv(\"input/cleaned_sp.csv\")\n",
    "test_data = pd.read_csv(\"input/cleaned_test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "en.columns = ['s1', 's2', 'label']\n",
    "# split dataset\n",
    "msk = np.random.rand(len(en)) < 0.8\n",
    "en_train = en[msk]\n",
    "en_valid = en[~msk]\n",
    "en_all_sents = en['s1'].tolist() + en['s2'].tolist()\n",
    "\n",
    "# dataset\n",
    "en_trainDS = myDS(en_train, en_all_sents)\n",
    "en_validDS = myDS(en_valid, en_all_sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp.columns = ['s1', 's2', 'label']\n",
    "# split dataset\n",
    "msk = np.random.rand(len(sp)) < 0.8\n",
    "sp_train = sp[msk]\n",
    "sp_valid = sp[~msk]\n",
    "sp_all_sents = sp['s1'].tolist() + sp['s2'].tolist()\n",
    "\n",
    "# dataset\n",
    "sp_trainDS = myDS(sp_train, sp_all_sents)\n",
    "sp_validDS = myDS(sp_valid, sp_all_sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_embed_dict = load_embed('input/en_embed.pkl')\n",
    "sp_embed_dict = load_embed('input/sp_embed.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 300\n",
    "en_embed_list = []\n",
    "for word in en_validDS.vocab._id2word:\n",
    "    en_embed_list.append(en_embed_dict[word])\n",
    "en_vocab_size = len(en_embed_list)\n",
    "    \n",
    "\n",
    "sp_embed_list = []\n",
    "for word in sp_trainDS.vocab._id2word:\n",
    "    sp_embed_list.append(sp_embed_dict[word])\n",
    "sp_vocab_size = len(sp_embed_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Input: English and Spanish embed list\n",
    "Output: English and Spanish aligned Embedding weight\n",
    "\"\"\"\n",
    "def align_embeddings(en_embed_list, sp_embed_list, embed_size):\n",
    "    print('English Vocab Size:{}, Spanish Vocab Size:{}'.format(len(en_embed_list), len(sp_embed_list)))\n",
    "    dif = abs(len(en_embed_list) - len(sp_embed_list))\n",
    "    compensate = []\n",
    "    for i in range(dif):\n",
    "        compensate.append(np.zeros(embed_size))\n",
    "    # shorter one aligned to longer one\n",
    "    if len(en_embed_list) < len(sp_embed_list):\n",
    "        en_embed_list.extend(compensate)\n",
    "    else: sp_embed_list.extend(compensate)\n",
    "    \n",
    "    if len(en_embed_list) == len(sp_embed_list):\n",
    "        print('-> Aligned to', len(en_embed_list))\n",
    "    \n",
    "    en_weight = nn.Parameter(torch.from_numpy(np.array(en_embed_list)).type(torch.FloatTensor), requires_grad = False)\n",
    "    sp_weight = nn.Parameter(torch.from_numpy(np.array(sp_embed_list)).type(torch.FloatTensor), requires_grad = False)\n",
    "\n",
    "    return en_weight, sp_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English Vocab Size:2685, Spanish Vocab Size:4101\n",
      "-> Aligned to 4101\n"
     ]
    }
   ],
   "source": [
    "aligned_size = max(en_vocab_size,sp_vocab_size)\n",
    "en_embedding = nn.Embedding(aligned_size, embed_size)\n",
    "sp_embedding = nn.Embedding(aligned_size, embed_size)\n",
    "\n",
    "en_embedding.weight, sp_embedding.weight = align_embeddings(en_embed_list, sp_embed_list, config['model']['embed_size'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Siamese_lstm(\n",
      "  (encoder): LSTMEncoder(\n",
      "    (embedding): Embedding(4101, 300)\n",
      "    (lstm): LSTM(300, 150, dropout=0.5)\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.1)\n",
      "    (1): Linear(in_features=600, out_features=100, bias=True)\n",
      "    (2): Tanh()\n",
      "    (3): Dropout(p=0.1)\n",
      "    (4): Linear(in_features=100, out_features=2, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "config['embedding_matrix'] = en_embedding\n",
    "# model\n",
    "siamese_en = Siamese_lstm(config)\n",
    "print(siamese_en)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('encoder.embedding.weight',\n",
       "              tensor([[ 0.0545, -0.8301,  0.1130,  ...,  0.3013, -0.2740,  0.7586],\n",
       "                      [ 0.2216, -0.8387,  0.4687,  ...,  0.8663,  0.3699,  0.2862],\n",
       "                      [ 0.8881,  0.5238,  0.6115,  ..., -0.7655,  0.5403, -0.6247],\n",
       "                      ...,\n",
       "                      [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "                      [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "                      [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]])),\n",
       "             ('encoder.lstm.weight_ih_l0',\n",
       "              tensor([[ 0.0138, -0.0523, -0.0013,  ..., -0.0419,  0.0260,  0.0701],\n",
       "                      [-0.0621,  0.0590, -0.0237,  ..., -0.0580,  0.0659,  0.0563],\n",
       "                      [-0.0009, -0.0701, -0.0804,  ..., -0.0771, -0.0558,  0.0595],\n",
       "                      ...,\n",
       "                      [ 0.0055,  0.0710,  0.0561,  ..., -0.0546,  0.0438,  0.0002],\n",
       "                      [ 0.0469,  0.0047, -0.0627,  ..., -0.0408,  0.0040, -0.0698],\n",
       "                      [ 0.0649,  0.0323,  0.0042,  ..., -0.0351, -0.0169,  0.0128]])),\n",
       "             ('encoder.lstm.weight_hh_l0',\n",
       "              tensor([[ 0.0299,  0.0620, -0.0594,  ..., -0.0482, -0.0422, -0.0280],\n",
       "                      [ 0.0232, -0.0562,  0.0325,  ...,  0.0344, -0.0667, -0.0436],\n",
       "                      [ 0.0613,  0.0733, -0.0231,  ..., -0.0044,  0.0727,  0.0770],\n",
       "                      ...,\n",
       "                      [ 0.0440,  0.0214, -0.0760,  ...,  0.0000,  0.0113,  0.0535],\n",
       "                      [-0.0673, -0.0485, -0.0661,  ...,  0.0233,  0.0065, -0.0121],\n",
       "                      [ 0.0295, -0.0134, -0.0428,  ...,  0.0302,  0.0616, -0.0508]])),\n",
       "             ('encoder.lstm.bias_ih_l0',\n",
       "              tensor([-0.0070,  0.0796, -0.0709, -0.0233,  0.0315,  0.0021, -0.0441,  0.0236,\n",
       "                      -0.0365, -0.0018,  0.0168,  0.0671, -0.0584, -0.0557, -0.0084, -0.0467,\n",
       "                       0.0059, -0.0060, -0.0646, -0.0199,  0.0220,  0.0008,  0.0658,  0.0003,\n",
       "                       0.0561, -0.0228, -0.0029,  0.0304,  0.0472,  0.0484, -0.0744,  0.0139,\n",
       "                      -0.0701,  0.0325, -0.0216, -0.0205, -0.0143,  0.0403,  0.0050, -0.0159,\n",
       "                       0.0054, -0.0759,  0.0090,  0.0247, -0.0433,  0.0568, -0.0165,  0.0612,\n",
       "                      -0.0216,  0.0748, -0.0359,  0.0609, -0.0523,  0.0172, -0.0798, -0.0116,\n",
       "                       0.0681,  0.0263,  0.0481, -0.0359, -0.0390, -0.0070, -0.0268,  0.0350,\n",
       "                       0.0173, -0.0760, -0.0047,  0.0284, -0.0510,  0.0434,  0.0538, -0.0630,\n",
       "                      -0.0686, -0.0377,  0.0163,  0.0712,  0.0413, -0.0070, -0.0051, -0.0096,\n",
       "                       0.0310, -0.0502, -0.0041, -0.0357,  0.0417, -0.0504, -0.0656, -0.0059,\n",
       "                      -0.0654, -0.0258,  0.0137, -0.0137,  0.0693,  0.0463,  0.0393, -0.0286,\n",
       "                       0.0762, -0.0187,  0.0681, -0.0447, -0.0544,  0.0334, -0.0481,  0.0475,\n",
       "                       0.0464, -0.0444, -0.0607,  0.0703,  0.0220, -0.0568,  0.0204, -0.0127,\n",
       "                       0.0161,  0.0409, -0.0474, -0.0391, -0.0536,  0.0210,  0.0295, -0.0104,\n",
       "                       0.0232,  0.0077, -0.0278,  0.0570,  0.0470, -0.0749, -0.0047,  0.0561,\n",
       "                       0.0770,  0.0030, -0.0511,  0.0413, -0.0773, -0.0755, -0.0030,  0.0344,\n",
       "                      -0.0114,  0.0339, -0.0281, -0.0662, -0.0242, -0.0389,  0.0288,  0.0653,\n",
       "                      -0.0524,  0.0637, -0.0165, -0.0583,  0.0152,  0.0576, -0.0553,  0.0656,\n",
       "                       0.0117, -0.0095, -0.0654, -0.0472,  0.0056, -0.0109, -0.0383,  0.0360,\n",
       "                      -0.0738, -0.0723,  0.0259,  0.0459,  0.0291,  0.0710,  0.0648,  0.0695,\n",
       "                      -0.0777, -0.0244, -0.0074,  0.0046,  0.0099, -0.0661, -0.0155,  0.0693,\n",
       "                       0.0713, -0.0615,  0.0025, -0.0016, -0.0235, -0.0439, -0.0645,  0.0049,\n",
       "                       0.0278, -0.0632, -0.0501,  0.0721, -0.0092,  0.0516,  0.0764, -0.0192,\n",
       "                       0.0048,  0.0804,  0.0709,  0.0388, -0.0200, -0.0204, -0.0493,  0.0454,\n",
       "                      -0.0334,  0.0193,  0.0369, -0.0627, -0.0084, -0.0243,  0.0519, -0.0514,\n",
       "                       0.0633,  0.0204, -0.0452, -0.0114,  0.0717, -0.0160,  0.0770,  0.0518,\n",
       "                      -0.0448, -0.0182, -0.0505,  0.0477,  0.0583,  0.0284,  0.0392,  0.0698,\n",
       "                      -0.0382,  0.0561,  0.0008, -0.0769,  0.0615,  0.0342,  0.0301,  0.0150,\n",
       "                      -0.0016, -0.0677, -0.0635,  0.0066,  0.0688, -0.0448, -0.0487,  0.0235,\n",
       "                      -0.0104, -0.0075,  0.0568, -0.0210, -0.0465, -0.0100,  0.0083, -0.0671,\n",
       "                      -0.0711, -0.0652, -0.0132, -0.0299, -0.0238,  0.0783, -0.0026, -0.0416,\n",
       "                      -0.0657,  0.0745,  0.0784, -0.0183, -0.0073,  0.0184, -0.0269,  0.0751,\n",
       "                       0.0617,  0.0185, -0.0216,  0.0437, -0.0497,  0.0387,  0.0079,  0.0729,\n",
       "                       0.0559, -0.0497, -0.0519, -0.0228, -0.0588, -0.0316,  0.0027,  0.0691,\n",
       "                       0.0242, -0.0780, -0.0165,  0.0136,  0.0661, -0.0538, -0.0487, -0.0358,\n",
       "                      -0.0120,  0.0769,  0.0623,  0.0547,  0.0116,  0.0158,  0.0674,  0.0615,\n",
       "                      -0.0333, -0.0193,  0.0142,  0.0524,  0.0010,  0.0434, -0.0180, -0.0803,\n",
       "                       0.0246,  0.0633, -0.0167, -0.0275, -0.0471, -0.0167,  0.0422,  0.0276,\n",
       "                      -0.0684, -0.0627,  0.0157,  0.0146, -0.0021, -0.0675,  0.0282, -0.0243,\n",
       "                      -0.0050, -0.0055,  0.0683, -0.0089,  0.0134, -0.0550, -0.0634, -0.0556,\n",
       "                      -0.0599,  0.0288, -0.0276,  0.0303, -0.0773, -0.0586, -0.0346, -0.0043,\n",
       "                       0.0273, -0.0506, -0.0573, -0.0612, -0.0527,  0.0567,  0.0110, -0.0096,\n",
       "                      -0.0035,  0.0673,  0.0105,  0.0711,  0.0297,  0.0769,  0.0620,  0.0269,\n",
       "                       0.0161,  0.0712, -0.0391,  0.0132, -0.0308,  0.0630, -0.0601, -0.0603,\n",
       "                       0.0137, -0.0453,  0.0603,  0.0262, -0.0680,  0.0161, -0.0232,  0.0096,\n",
       "                       0.0404, -0.0663, -0.0718,  0.0174, -0.0157,  0.0678, -0.0717,  0.0566,\n",
       "                       0.0414, -0.0550, -0.0254, -0.0753, -0.0271, -0.0793,  0.0075, -0.0086,\n",
       "                      -0.0235,  0.0571, -0.0069, -0.0604, -0.0293,  0.0778,  0.0783, -0.0431,\n",
       "                      -0.0681, -0.0210,  0.0350,  0.0481,  0.0454, -0.0575, -0.0275, -0.0194,\n",
       "                       0.0691, -0.0464, -0.0370, -0.0031, -0.0504,  0.0430,  0.0353,  0.0388,\n",
       "                       0.0690, -0.0066, -0.0196,  0.0565,  0.0767,  0.0001,  0.0734,  0.0406,\n",
       "                       0.0569,  0.0220, -0.0239, -0.0256, -0.0472,  0.0142, -0.0602,  0.0379,\n",
       "                       0.0411,  0.0489, -0.0462,  0.0713, -0.0270,  0.0805,  0.0590,  0.0459,\n",
       "                       0.0521,  0.0048,  0.0386, -0.0658,  0.0213,  0.0560,  0.0402, -0.0460,\n",
       "                       0.0793,  0.0306,  0.0633,  0.0429, -0.0788, -0.0694, -0.0236, -0.0263,\n",
       "                      -0.0265,  0.0312, -0.0256, -0.0637, -0.0604, -0.0493, -0.0043, -0.0189,\n",
       "                       0.0612,  0.0161,  0.0575, -0.0422,  0.0664, -0.0091, -0.0066, -0.0676,\n",
       "                      -0.0147, -0.0728, -0.0348, -0.0364,  0.0410,  0.0207, -0.0563,  0.0597,\n",
       "                       0.0654, -0.0199, -0.0652,  0.0328, -0.0148,  0.0433, -0.0535, -0.0647,\n",
       "                      -0.0787,  0.0404, -0.0725, -0.0793, -0.0542, -0.0082, -0.0416,  0.0151,\n",
       "                       0.0725,  0.0031,  0.0315,  0.0028, -0.0526,  0.0739, -0.0087,  0.0140,\n",
       "                       0.0281, -0.0212, -0.0691,  0.0021, -0.0108, -0.0172, -0.0784,  0.0338,\n",
       "                       0.0369,  0.0747,  0.0533, -0.0436, -0.0086, -0.0218,  0.0638,  0.0770,\n",
       "                       0.0523, -0.0375,  0.0693,  0.0572,  0.0744, -0.0028,  0.0121, -0.0123,\n",
       "                      -0.0035, -0.0639, -0.0291,  0.0805, -0.0273, -0.0242, -0.0737,  0.0179,\n",
       "                      -0.0194, -0.0264,  0.0171, -0.0748, -0.0281,  0.0505, -0.0271, -0.0261,\n",
       "                       0.0109,  0.0494, -0.0754,  0.0511, -0.0261,  0.0396, -0.0336,  0.0037,\n",
       "                       0.0250, -0.0238,  0.0086,  0.0033,  0.0309, -0.0559,  0.0425, -0.0311,\n",
       "                       0.0794,  0.0434,  0.0216, -0.0369, -0.0036,  0.0038, -0.0028,  0.0755,\n",
       "                      -0.0796,  0.0135,  0.0393,  0.0223,  0.0594,  0.0019, -0.0285, -0.0244,\n",
       "                      -0.0796,  0.0164,  0.0004,  0.0091,  0.0283, -0.0664,  0.0163,  0.0456,\n",
       "                      -0.0260, -0.0735, -0.0035,  0.0592, -0.0713, -0.0252,  0.0310, -0.0214,\n",
       "                       0.0694, -0.0414,  0.0711, -0.0400,  0.0763,  0.0802,  0.0499,  0.0395,\n",
       "                      -0.0125,  0.0079, -0.0687, -0.0812, -0.0008, -0.0670, -0.0153,  0.0367])),\n",
       "             ('encoder.lstm.bias_hh_l0',\n",
       "              tensor([ 0.0275, -0.0783,  0.0096,  0.0324, -0.0030, -0.0063, -0.0399,  0.0092,\n",
       "                       0.0488,  0.0747,  0.0803,  0.0019, -0.0195,  0.0134,  0.0316, -0.0628,\n",
       "                       0.0207,  0.0225, -0.0166, -0.0124, -0.0453,  0.0589,  0.0667, -0.0598,\n",
       "                      -0.0441, -0.0620, -0.0792, -0.0117, -0.0489, -0.0549, -0.0458, -0.0714,\n",
       "                       0.0129, -0.0168, -0.0365,  0.0438, -0.0696,  0.0702,  0.0255, -0.0078,\n",
       "                      -0.0079,  0.0184, -0.0030, -0.0555,  0.0422, -0.0016,  0.0663,  0.0223,\n",
       "                       0.0196, -0.0738,  0.0678, -0.0453,  0.0600,  0.0568,  0.0026, -0.0748,\n",
       "                      -0.0444,  0.0505, -0.0445, -0.0299,  0.0357,  0.0310, -0.0197,  0.0374,\n",
       "                       0.0510,  0.0267, -0.0708,  0.0552, -0.0704,  0.0797,  0.0792, -0.0259,\n",
       "                       0.0616, -0.0686,  0.0537,  0.0371,  0.0031,  0.0396,  0.0411, -0.0116,\n",
       "                      -0.0445,  0.0477, -0.0424, -0.0813, -0.0320, -0.0047, -0.0462,  0.0219,\n",
       "                      -0.0167, -0.0259,  0.0385,  0.0357, -0.0708, -0.0653,  0.0444, -0.0099,\n",
       "                      -0.0459, -0.0755, -0.0709,  0.0462,  0.0491, -0.0409,  0.0011, -0.0032,\n",
       "                       0.0704, -0.0494, -0.0773,  0.0516, -0.0765, -0.0104,  0.0492, -0.0120,\n",
       "                       0.0699, -0.0314, -0.0223,  0.0813, -0.0224,  0.0735, -0.0747,  0.0118,\n",
       "                       0.0668,  0.0634,  0.0104, -0.0533,  0.0628, -0.0231, -0.0432,  0.0598,\n",
       "                       0.0015,  0.0229, -0.0074,  0.0247, -0.0708,  0.0655, -0.0554,  0.0550,\n",
       "                      -0.0369, -0.0690, -0.0361, -0.0690,  0.0229,  0.0759, -0.0506, -0.0311,\n",
       "                      -0.0111,  0.0306,  0.0775, -0.0288,  0.0714,  0.0043, -0.0225, -0.0257,\n",
       "                       0.0772,  0.0122,  0.0709, -0.0690, -0.0349, -0.0327, -0.0307, -0.0497,\n",
       "                       0.0353,  0.0809,  0.0118,  0.0485, -0.0648, -0.0500,  0.0064,  0.0306,\n",
       "                      -0.0358, -0.0088, -0.0200, -0.0759,  0.0353, -0.0510, -0.0012,  0.0023,\n",
       "                      -0.0079, -0.0235,  0.0285, -0.0146,  0.0163, -0.0500, -0.0345, -0.0286,\n",
       "                      -0.0162,  0.0504, -0.0292,  0.0641,  0.0100, -0.0702,  0.0018, -0.0535,\n",
       "                       0.0809,  0.0054, -0.0313,  0.0733,  0.0092,  0.0420,  0.0795, -0.0754,\n",
       "                      -0.0258, -0.0285, -0.0156,  0.0070, -0.0297, -0.0512,  0.0736,  0.0788,\n",
       "                       0.0213, -0.0257,  0.0249,  0.0393,  0.0750, -0.0303, -0.0160, -0.0362,\n",
       "                      -0.0483, -0.0319,  0.0172, -0.0217,  0.0264,  0.0413, -0.0612, -0.0581,\n",
       "                      -0.0061,  0.0130,  0.0176,  0.0502,  0.0758,  0.0298,  0.0384, -0.0458,\n",
       "                       0.0118, -0.0527,  0.0347,  0.0592, -0.0182, -0.0677, -0.0437,  0.0473,\n",
       "                       0.0365,  0.0222, -0.0723,  0.0097,  0.0314,  0.0471, -0.0535, -0.0513,\n",
       "                      -0.0283,  0.0757,  0.0362,  0.0093, -0.0295,  0.0098,  0.0292, -0.0696,\n",
       "                       0.0655,  0.0038, -0.0535, -0.0033, -0.0212, -0.0649,  0.0801,  0.0161,\n",
       "                      -0.0807, -0.0809, -0.0489, -0.0470, -0.0615, -0.0265, -0.0773,  0.0213,\n",
       "                       0.0231,  0.0043,  0.0118, -0.0120, -0.0051,  0.0644, -0.0576,  0.0763,\n",
       "                       0.0341, -0.0262, -0.0455,  0.0422,  0.0590,  0.0787,  0.0087, -0.0730,\n",
       "                      -0.0555,  0.0297,  0.0390,  0.0679,  0.0339,  0.0285, -0.0220,  0.0355,\n",
       "                       0.0528, -0.0631,  0.0434,  0.0628,  0.0388, -0.0286, -0.0651, -0.0052,\n",
       "                      -0.0458,  0.0240, -0.0677,  0.0638,  0.0441,  0.0320, -0.0165,  0.0240,\n",
       "                      -0.0421,  0.0415, -0.0266, -0.0069,  0.0464,  0.0673, -0.0405,  0.0569,\n",
       "                      -0.0255,  0.0259, -0.0435, -0.0445,  0.0356,  0.0624, -0.0400, -0.0106,\n",
       "                       0.0240, -0.0136, -0.0198, -0.0388, -0.0481, -0.0539, -0.0405,  0.0753,\n",
       "                       0.0362, -0.0815, -0.0176, -0.0309,  0.0788, -0.0424,  0.0441, -0.0527,\n",
       "                       0.0783, -0.0053, -0.0408, -0.0452,  0.0423, -0.0586, -0.0350,  0.0010,\n",
       "                      -0.0073,  0.0546, -0.0804,  0.0301,  0.0494, -0.0505,  0.0150, -0.0578,\n",
       "                      -0.0655,  0.0629,  0.0178,  0.0144, -0.0555, -0.0423,  0.0248, -0.0635,\n",
       "                      -0.0573,  0.0143,  0.0208,  0.0663, -0.0440,  0.0758,  0.0631,  0.0307,\n",
       "                      -0.0110,  0.0810,  0.0163,  0.0018,  0.0306, -0.0389,  0.0599, -0.0351,\n",
       "                      -0.0074, -0.0214, -0.0584, -0.0777,  0.0360,  0.0167,  0.0183, -0.0145,\n",
       "                      -0.0293,  0.0485,  0.0486, -0.0398,  0.0683, -0.0568,  0.0612, -0.0202,\n",
       "                       0.0713,  0.0606,  0.0002,  0.0082,  0.0323, -0.0575,  0.0593,  0.0246,\n",
       "                      -0.0812, -0.0118,  0.0053,  0.0302,  0.0024,  0.0452,  0.0646,  0.0509,\n",
       "                      -0.0254,  0.0323, -0.0201, -0.0057,  0.0655, -0.0201, -0.0250,  0.0449,\n",
       "                       0.0743,  0.0373, -0.0195, -0.0140,  0.0773, -0.0414, -0.0342,  0.0504,\n",
       "                       0.0211, -0.0623, -0.0588, -0.0667,  0.0274,  0.0458,  0.0420, -0.0799,\n",
       "                       0.0066,  0.0094,  0.0722, -0.0207, -0.0808,  0.0783,  0.0447, -0.0757,\n",
       "                       0.0495,  0.0360, -0.0188,  0.0136,  0.0318,  0.0138, -0.0198,  0.0181,\n",
       "                       0.0584,  0.0058, -0.0065,  0.0727, -0.0358, -0.0804, -0.0807,  0.0196,\n",
       "                       0.0487,  0.0374,  0.0336, -0.0468, -0.0457, -0.0386, -0.0774, -0.0464,\n",
       "                      -0.0636, -0.0693, -0.0522, -0.0286,  0.0261, -0.0119, -0.0598,  0.0164,\n",
       "                      -0.0410, -0.0147, -0.0084,  0.0522,  0.0279,  0.0077, -0.0400,  0.0548,\n",
       "                       0.0768,  0.0052,  0.0734, -0.0732, -0.0096, -0.0627, -0.0660, -0.0435,\n",
       "                       0.0696,  0.0719, -0.0101,  0.0564, -0.0671, -0.0360, -0.0220, -0.0265,\n",
       "                       0.0594, -0.0101,  0.0096, -0.0048, -0.0535, -0.0563, -0.0002, -0.0176,\n",
       "                       0.0297,  0.0458,  0.0489, -0.0093, -0.0588, -0.0145,  0.0736,  0.0499,\n",
       "                       0.0101,  0.0121, -0.0348,  0.0111,  0.0071,  0.0287, -0.0303, -0.0565,\n",
       "                       0.0067, -0.0160,  0.0429, -0.0767, -0.0547, -0.0390,  0.0665,  0.0082,\n",
       "                       0.0298, -0.0561, -0.0755, -0.0229, -0.0754, -0.0468, -0.0211,  0.0154,\n",
       "                       0.0524, -0.0210,  0.0363, -0.0622,  0.0142, -0.0576, -0.0090, -0.0276,\n",
       "                       0.0126, -0.0387,  0.0778, -0.0332, -0.0272, -0.0216,  0.0190, -0.0788,\n",
       "                      -0.0602,  0.0682, -0.0727, -0.0622,  0.0321, -0.0051, -0.0521, -0.0561,\n",
       "                       0.0503,  0.0010, -0.0127,  0.0439,  0.0757, -0.0327,  0.0483, -0.0185,\n",
       "                       0.0425,  0.0788, -0.0221,  0.0789, -0.0246, -0.0669, -0.0107,  0.0748,\n",
       "                      -0.0216, -0.0810,  0.0084, -0.0300, -0.0184,  0.0024, -0.0320, -0.0211,\n",
       "                      -0.0631,  0.0691,  0.0115, -0.0480, -0.0691,  0.0432,  0.0782, -0.0366])),\n",
       "             ('classifier.1.weight',\n",
       "              tensor([[ 0.0400, -0.0033,  0.0164,  ..., -0.0221, -0.0059,  0.0142],\n",
       "                      [-0.0119,  0.0075, -0.0407,  ..., -0.0399, -0.0316, -0.0351],\n",
       "                      [-0.0368,  0.0378,  0.0095,  ..., -0.0060, -0.0340,  0.0253],\n",
       "                      ...,\n",
       "                      [-0.0197,  0.0179,  0.0205,  ..., -0.0030,  0.0312,  0.0087],\n",
       "                      [ 0.0071, -0.0401,  0.0282,  ...,  0.0149, -0.0136, -0.0038],\n",
       "                      [-0.0059, -0.0242,  0.0280,  ..., -0.0253,  0.0142, -0.0079]])),\n",
       "             ('classifier.1.bias',\n",
       "              tensor([ 0.0304,  0.0057, -0.0118, -0.0066,  0.0136, -0.0027,  0.0244, -0.0089,\n",
       "                      -0.0166,  0.0113,  0.0025,  0.0108, -0.0331,  0.0066, -0.0207, -0.0335,\n",
       "                       0.0267, -0.0179, -0.0266,  0.0183,  0.0124,  0.0085,  0.0199,  0.0258,\n",
       "                       0.0079,  0.0383,  0.0385, -0.0018,  0.0075, -0.0404,  0.0142, -0.0039,\n",
       "                      -0.0028,  0.0213, -0.0177,  0.0219,  0.0407,  0.0228,  0.0359, -0.0299,\n",
       "                       0.0148,  0.0293, -0.0084,  0.0088, -0.0092,  0.0100,  0.0203, -0.0313,\n",
       "                      -0.0050,  0.0189, -0.0092, -0.0134,  0.0269, -0.0322, -0.0302,  0.0291,\n",
       "                       0.0176,  0.0182,  0.0141, -0.0070, -0.0274, -0.0331,  0.0239,  0.0259,\n",
       "                      -0.0224, -0.0175, -0.0362,  0.0392, -0.0275,  0.0264, -0.0092, -0.0195,\n",
       "                      -0.0023, -0.0143,  0.0302, -0.0038,  0.0389,  0.0279, -0.0377, -0.0136,\n",
       "                      -0.0389, -0.0140,  0.0268,  0.0365,  0.0099, -0.0184, -0.0323,  0.0023,\n",
       "                       0.0246,  0.0313,  0.0378,  0.0221,  0.0395,  0.0148,  0.0221, -0.0151,\n",
       "                       0.0224, -0.0406,  0.0118, -0.0396])),\n",
       "             ('classifier.4.weight',\n",
       "              tensor([[ 0.0468,  0.0763,  0.0649, -0.0442,  0.0179,  0.0748, -0.0017,  0.0411,\n",
       "                       -0.0128, -0.0930,  0.0146,  0.0513,  0.0838,  0.0230, -0.0696, -0.0283,\n",
       "                        0.0177, -0.0179,  0.0017,  0.0791,  0.0200,  0.0752,  0.0826,  0.0615,\n",
       "                        0.0905, -0.0746,  0.0778,  0.0633,  0.0947,  0.0117, -0.0312, -0.0457,\n",
       "                        0.0256, -0.0278, -0.0747,  0.0780,  0.0387,  0.0812,  0.0032,  0.0275,\n",
       "                       -0.0345, -0.0416, -0.0715,  0.0639, -0.0093,  0.0742,  0.0299,  0.0437,\n",
       "                       -0.0196, -0.0543, -0.0960,  0.0804,  0.0830,  0.0362,  0.0787, -0.0366,\n",
       "                       -0.0643,  0.0359, -0.0039, -0.0226,  0.0062,  0.0055,  0.0312, -0.0011,\n",
       "                       -0.0854,  0.0501, -0.0999,  0.0447,  0.0432, -0.0400, -0.0941, -0.0925,\n",
       "                        0.0531,  0.0573,  0.0354, -0.0726,  0.0766,  0.0836,  0.0164, -0.0369,\n",
       "                        0.0603,  0.0709, -0.0079, -0.0157, -0.0834, -0.0825,  0.0249,  0.0784,\n",
       "                       -0.0454,  0.0026,  0.0055, -0.0361, -0.0888,  0.0204, -0.0773,  0.0331,\n",
       "                        0.0602,  0.0442, -0.0799, -0.0372],\n",
       "                      [ 0.0908, -0.0345,  0.0016, -0.0326, -0.0950, -0.0826, -0.0588,  0.0618,\n",
       "                       -0.0660, -0.0112, -0.0635,  0.0071, -0.0562,  0.0013, -0.0941, -0.0250,\n",
       "                        0.0928, -0.0375, -0.0210,  0.0494,  0.0227, -0.0563, -0.0299,  0.0299,\n",
       "                        0.0389, -0.0477,  0.0613, -0.0521,  0.0516, -0.0640,  0.0210, -0.0578,\n",
       "                       -0.0424,  0.0324,  0.0921,  0.0383, -0.0323, -0.0651, -0.0257,  0.0001,\n",
       "                        0.0057,  0.0959, -0.0870, -0.0488,  0.0140,  0.0143, -0.0198, -0.0675,\n",
       "                        0.0725,  0.0153, -0.0093, -0.0358,  0.0625, -0.0142, -0.0534,  0.0539,\n",
       "                       -0.0375, -0.0899,  0.0439, -0.0834,  0.0075,  0.0604, -0.0560,  0.0105,\n",
       "                       -0.0052, -0.0164,  0.0797,  0.0820, -0.0261,  0.0031, -0.0519, -0.0134,\n",
       "                       -0.0051,  0.0001,  0.0315,  0.0962,  0.0853, -0.0663, -0.0074, -0.0382,\n",
       "                        0.0813, -0.0771, -0.0329, -0.0397,  0.0841,  0.0491,  0.0752,  0.0498,\n",
       "                        0.0876,  0.0391, -0.0026,  0.0376, -0.0063, -0.0324,  0.0280,  0.0641,\n",
       "                        0.0140, -0.0814, -0.0373,  0.0757]])),\n",
       "             ('classifier.4.bias', tensor([-0.0635, -0.0875]))])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "siamese_en.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('encoder.embedding.weight',\n",
       "              tensor([[ 0.0545, -0.8301,  0.1130,  ...,  0.3013, -0.2740,  0.7586],\n",
       "                      [ 0.2216, -0.8387,  0.4687,  ...,  0.8663,  0.3699,  0.2862],\n",
       "                      [ 0.8881,  0.5238,  0.6115,  ..., -0.7655,  0.5403, -0.6247],\n",
       "                      ...,\n",
       "                      [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "                      [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "                      [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]])),\n",
       "             ('encoder.lstm.weight_ih_l0',\n",
       "              tensor([[ 0.0361, -0.0769,  0.0482,  ..., -0.0454,  0.0273, -0.0062],\n",
       "                      [ 0.0088, -0.0274,  0.0335,  ..., -0.0576, -0.0711, -0.0272],\n",
       "                      [ 0.0027,  0.0797,  0.0173,  ..., -0.0505,  0.0132,  0.0350],\n",
       "                      ...,\n",
       "                      [ 0.0568,  0.0336, -0.0606,  ...,  0.0338,  0.0600,  0.0458],\n",
       "                      [ 0.0819,  0.0404,  0.0431,  ...,  0.0195,  0.0116, -0.0550],\n",
       "                      [-0.0057, -0.0024,  0.0145,  ..., -0.0345, -0.0801,  0.0809]])),\n",
       "             ('encoder.lstm.weight_hh_l0',\n",
       "              tensor([[-0.0160,  0.0629,  0.0072,  ...,  0.0177,  0.0148, -0.0361],\n",
       "                      [-0.0669, -0.0654, -0.0525,  ...,  0.0201,  0.0646, -0.0808],\n",
       "                      [ 0.0015, -0.0166, -0.0422,  ..., -0.0361,  0.0285,  0.0175],\n",
       "                      ...,\n",
       "                      [-0.0569, -0.0634,  0.0157,  ..., -0.0672, -0.0285, -0.0049],\n",
       "                      [ 0.0087, -0.0254,  0.0373,  ...,  0.0793,  0.0251,  0.0789],\n",
       "                      [ 0.0819,  0.0380,  0.0458,  ...,  0.0002,  0.0421, -0.0123]])),\n",
       "             ('encoder.lstm.bias_ih_l0',\n",
       "              tensor([ 0.0204,  0.0322,  0.0696,  0.0044,  0.0747,  0.0081, -0.0635,  0.0961,\n",
       "                      -0.0410, -0.0419,  0.0635,  0.0188,  0.0821,  0.0578,  0.0345,  0.0330,\n",
       "                       0.0018,  0.0275,  0.0485,  0.1274,  0.0564, -0.0138,  0.0415,  0.0563,\n",
       "                       0.0716,  0.0606, -0.0260, -0.0224,  0.0014,  0.0191,  0.0512, -0.0305,\n",
       "                       0.0641,  0.0258,  0.0585,  0.0348,  0.0069,  0.0540, -0.0546, -0.0473,\n",
       "                      -0.0210, -0.0237,  0.0504,  0.0358,  0.0285, -0.0195,  0.0809, -0.0075,\n",
       "                      -0.0685,  0.0077, -0.0744, -0.0324,  0.0032, -0.0109,  0.0156,  0.0211,\n",
       "                      -0.0125,  0.0325,  0.0954, -0.0453,  0.0374, -0.0720,  0.0023, -0.0611,\n",
       "                       0.0250,  0.0261, -0.0211, -0.0467,  0.0448,  0.0538,  0.0266, -0.0665,\n",
       "                       0.1066,  0.0760, -0.0056,  0.0148, -0.0115, -0.0711,  0.0266, -0.0544,\n",
       "                       0.0863, -0.0108, -0.0072, -0.0755, -0.0569, -0.0238,  0.0492,  0.0102,\n",
       "                       0.0705,  0.0272, -0.0398,  0.0285,  0.0287, -0.0187, -0.0263,  0.0422,\n",
       "                       0.0354, -0.0191, -0.0827, -0.0497,  0.0450,  0.0368,  0.0679, -0.0141,\n",
       "                      -0.0054, -0.0468,  0.1134,  0.0757,  0.0246, -0.0809,  0.0693,  0.0351,\n",
       "                       0.0245,  0.0541,  0.0683,  0.0696,  0.0021, -0.0077, -0.0147, -0.0548,\n",
       "                      -0.0599, -0.0314,  0.0837,  0.0105,  0.0049,  0.0693, -0.0684, -0.0639,\n",
       "                       0.0121, -0.0321,  0.0726, -0.0084,  0.0611, -0.0179,  0.1083, -0.0271,\n",
       "                       0.0736,  0.0367,  0.0199,  0.0641,  0.0182,  0.0500,  0.0443, -0.0466,\n",
       "                       0.0031,  0.0907,  0.0739,  0.0204, -0.0687, -0.0754,  0.0723,  0.0103,\n",
       "                       0.0332, -0.0391, -0.0509, -0.0428, -0.0842, -0.0382,  0.2058, -0.0341,\n",
       "                       0.0451, -0.0079,  0.0400,  0.0072, -0.0090, -0.0301,  0.0231,  0.0230,\n",
       "                       0.0265,  0.0073, -0.0826, -0.0086,  0.0085,  0.0660,  0.0396, -0.0124,\n",
       "                       0.0613, -0.0260,  0.0379, -0.0320, -0.0515, -0.0034,  0.0177, -0.0003,\n",
       "                       0.0522, -0.0463,  0.0496,  0.0145,  0.0443,  0.0182,  0.0245,  0.0043,\n",
       "                      -0.0574,  0.0575,  0.0185,  0.0267,  0.0395,  0.0494,  0.0326, -0.0010,\n",
       "                       0.0051,  0.0544, -0.0748, -0.0329,  0.0334,  0.0677,  0.0611,  0.0136,\n",
       "                      -0.0590,  0.0350, -0.0491,  0.0455,  0.0533,  0.0545,  0.0666,  0.0291,\n",
       "                      -0.0388,  0.0271,  0.0566,  0.0162, -0.0400, -0.0669, -0.0668,  0.0462,\n",
       "                       0.0085, -0.0898, -0.0348,  0.0128,  0.0278,  0.0477,  0.0796, -0.0691,\n",
       "                      -0.0071, -0.0210,  0.0200, -0.0339, -0.0153, -0.0874, -0.0773, -0.0660,\n",
       "                       0.0217,  0.0160, -0.0257,  0.0457,  0.0706, -0.0358,  0.0291, -0.0894,\n",
       "                       0.0754, -0.0441, -0.0117,  0.0145, -0.0125, -0.0735, -0.0831,  0.0566,\n",
       "                       0.0894,  0.0644, -0.0414,  0.0616, -0.0105,  0.0114, -0.0352,  0.0421,\n",
       "                       0.0318, -0.0610,  0.0426, -0.0434, -0.0143,  0.0463, -0.0677, -0.0430,\n",
       "                      -0.0079,  0.0500,  0.0631, -0.0288, -0.0064,  0.0565,  0.0665, -0.0568,\n",
       "                       0.0033,  0.0180,  0.0439,  0.0068,  0.0686,  0.0006, -0.0164,  0.0301,\n",
       "                      -0.0628, -0.0655,  0.0418, -0.0671, -0.0471, -0.0132, -0.0650,  0.0315,\n",
       "                      -0.0270, -0.0441, -0.0247, -0.0274,  0.0319, -0.0793,  0.0152, -0.0370,\n",
       "                      -0.0435, -0.0601,  0.0791, -0.0212,  0.1400,  0.0718,  0.0229,  0.0862,\n",
       "                       0.0710, -0.0036, -0.0223, -0.0423,  0.0560,  0.0748,  0.0791, -0.0578,\n",
       "                       0.0522,  0.0187,  0.0060, -0.0806, -0.0282,  0.0137, -0.0465, -0.0293,\n",
       "                       0.0158, -0.0429, -0.0279, -0.0777, -0.0215,  0.0174, -0.0429, -0.0001,\n",
       "                       0.0376,  0.0016, -0.0463,  0.0191, -0.0322,  0.0511, -0.0542, -0.0355,\n",
       "                      -0.0618,  0.0006,  0.0628,  0.0574, -0.0040, -0.0347,  0.0486,  0.0449,\n",
       "                      -0.0575,  0.0286,  0.0785, -0.0066, -0.0012,  0.0112,  0.0069, -0.0716,\n",
       "                       0.0261, -0.0668,  0.0172, -0.0019, -0.0124, -0.0746, -0.0189,  0.0840,\n",
       "                       0.0299, -0.0096, -0.0195, -0.0798, -0.0674,  0.0573, -0.0703,  0.0874,\n",
       "                      -0.0839,  0.0050, -0.0486,  0.0620,  0.0212,  0.0649,  0.0264,  0.0035,\n",
       "                       0.0988, -0.0128, -0.0191,  0.0015,  0.0129,  0.0026,  0.0553, -0.0487,\n",
       "                       0.0140,  0.0530,  0.0444,  0.0246,  0.0662,  0.0307, -0.0028,  0.0219,\n",
       "                       0.0665, -0.0432, -0.0601,  0.0476,  0.0155, -0.0790, -0.0210,  0.0551,\n",
       "                       0.0484,  0.0257,  0.0431,  0.0769,  0.0183, -0.0635, -0.0298, -0.0947,\n",
       "                      -0.0266,  0.0684,  0.0627,  0.0673,  0.0021, -0.0376, -0.0435,  0.0546,\n",
       "                      -0.0492, -0.0675,  0.0538, -0.0465, -0.0210, -0.0769, -0.0177,  0.0548,\n",
       "                      -0.0528,  0.0084,  0.0276,  0.0735, -0.0218, -0.0895,  0.0353,  0.0179,\n",
       "                       0.0586, -0.0682,  0.0790, -0.0542, -0.0091, -0.0711,  0.0132,  0.0420,\n",
       "                      -0.0259,  0.0398,  0.0291, -0.0785, -0.0712,  0.0401, -0.0124, -0.0688,\n",
       "                      -0.0872,  0.0131,  0.0260, -0.0437,  0.0950, -0.0796,  0.0288, -0.0627,\n",
       "                      -0.0645, -0.0551,  0.0435, -0.0025,  0.0569,  0.0306, -0.0747, -0.0153,\n",
       "                       0.0197,  0.0056,  0.0257,  0.0188,  0.0347, -0.0147,  0.0172, -0.0642,\n",
       "                      -0.0022,  0.0505,  0.0412,  0.0402,  0.0293,  0.0512,  0.0471,  0.0741,\n",
       "                      -0.0526, -0.0488, -0.0826,  0.0451,  0.0474,  0.0302,  0.0361, -0.0590,\n",
       "                       0.0260,  0.0525, -0.0453, -0.0269,  0.0486,  0.0523, -0.0068, -0.0856,\n",
       "                      -0.0453, -0.0675, -0.0227,  0.0219,  0.0129,  0.0255, -0.0209, -0.0549,\n",
       "                       0.0425, -0.0076, -0.0121,  0.0515, -0.0392, -0.0620,  0.0661,  0.0738,\n",
       "                       0.0351, -0.0399, -0.0469, -0.0284,  0.0695,  0.0593, -0.0601, -0.0828,\n",
       "                      -0.0697, -0.0063, -0.0142, -0.0084,  0.0399,  0.0037, -0.0603, -0.0565,\n",
       "                       0.0215,  0.0274, -0.0067, -0.0095,  0.0280, -0.0486, -0.0356, -0.0554,\n",
       "                       0.0405,  0.0188, -0.0728, -0.0005, -0.0816,  0.0260, -0.0439, -0.0401,\n",
       "                      -0.0829, -0.0218,  0.0014,  0.0238,  0.0688,  0.0009, -0.0473,  0.0247,\n",
       "                      -0.0585, -0.0124, -0.0006, -0.0064, -0.0151, -0.0555, -0.0073, -0.0093,\n",
       "                      -0.0725, -0.0595,  0.0418, -0.0606, -0.0717, -0.0496,  0.0191,  0.0556,\n",
       "                      -0.0489, -0.0658, -0.0175,  0.0289, -0.0200, -0.0369,  0.0747, -0.0841,\n",
       "                       0.1462,  0.0683, -0.0079, -0.0433, -0.0340,  0.0748, -0.0065,  0.0487,\n",
       "                       0.0186, -0.0191,  0.0236,  0.0423,  0.0019, -0.0021,  0.0853, -0.0855])),\n",
       "             ('encoder.lstm.bias_hh_l0',\n",
       "              tensor([ 0.0379,  0.0413, -0.0380, -0.0729,  0.0556, -0.0453, -0.0762, -0.0619,\n",
       "                       0.0254, -0.0788,  0.0981,  0.0059,  0.0202, -0.0738, -0.0740, -0.0228,\n",
       "                       0.0111,  0.0765,  0.0718,  0.0511, -0.0354, -0.0451,  0.0807,  0.0174,\n",
       "                       0.0792,  0.0571,  0.0048, -0.0444, -0.0119,  0.0522,  0.0398,  0.0174,\n",
       "                      -0.0123,  0.0086, -0.0118, -0.0046,  0.0755,  0.0413,  0.0305, -0.0532,\n",
       "                      -0.0623,  0.0706,  0.0454,  0.0003,  0.0555,  0.0254,  0.0766, -0.0265,\n",
       "                       0.0534, -0.0342, -0.0050, -0.0458, -0.0622, -0.0585, -0.0119,  0.0415,\n",
       "                       0.0578,  0.0762,  0.0020,  0.0708, -0.0404,  0.0438, -0.0655, -0.0302,\n",
       "                       0.0402, -0.0277, -0.0154,  0.0991, -0.0430, -0.0602, -0.0067, -0.0706,\n",
       "                      -0.0026, -0.0414,  0.0498,  0.0670,  0.0147,  0.0749,  0.0865, -0.0755,\n",
       "                      -0.0208,  0.0350, -0.0354,  0.0291,  0.0155, -0.0156, -0.0753,  0.0736,\n",
       "                       0.0714, -0.0882, -0.0596,  0.0419,  0.0559, -0.0130, -0.0145,  0.0678,\n",
       "                      -0.0513,  0.0133, -0.0418, -0.0586, -0.0129, -0.0123,  0.0544, -0.0565,\n",
       "                       0.0299, -0.0227,  0.0419,  0.0024, -0.0327, -0.0481, -0.0022, -0.0455,\n",
       "                       0.0150,  0.0629,  0.0406,  0.0538,  0.0200,  0.0358,  0.0786, -0.0520,\n",
       "                       0.0279,  0.0389, -0.0726, -0.0049,  0.0850,  0.0707, -0.0220, -0.0680,\n",
       "                       0.0134, -0.0441, -0.0681,  0.0165, -0.0356, -0.0121,  0.1150,  0.0724,\n",
       "                       0.0101,  0.0064, -0.0024, -0.0281,  0.0003, -0.0671, -0.0448,  0.0723,\n",
       "                       0.0588,  0.0370, -0.0118, -0.0283, -0.0276, -0.0503, -0.0100, -0.0012,\n",
       "                      -0.0329, -0.0932, -0.0537, -0.0903, -0.0507, -0.0681,  0.1440,  0.0329,\n",
       "                       0.0005, -0.0316,  0.0452, -0.0053, -0.0842,  0.0153, -0.0120, -0.0056,\n",
       "                      -0.0644, -0.0276, -0.0654,  0.0783, -0.0736, -0.0581, -0.0073, -0.0144,\n",
       "                      -0.0710,  0.0041,  0.0480, -0.0083,  0.0245,  0.0468,  0.0388,  0.0316,\n",
       "                      -0.0799, -0.0211,  0.0570, -0.0764, -0.0487,  0.0659,  0.0015, -0.0567,\n",
       "                      -0.0386, -0.0152,  0.0152,  0.0337, -0.0320,  0.0612,  0.0336, -0.0585,\n",
       "                      -0.0242, -0.0160, -0.0222,  0.0032,  0.0599, -0.0812,  0.0039,  0.0439,\n",
       "                      -0.0576, -0.0004,  0.0274,  0.0584, -0.0005, -0.0356,  0.0087, -0.0207,\n",
       "                       0.0545,  0.0127,  0.0601, -0.0602,  0.0479,  0.0622,  0.0467,  0.0028,\n",
       "                       0.0510, -0.0413,  0.0585,  0.0573,  0.0589, -0.0247, -0.0485,  0.0680,\n",
       "                      -0.0533,  0.0306,  0.0577, -0.0769,  0.0191, -0.0581, -0.0234,  0.0408,\n",
       "                       0.0525, -0.0848,  0.0048, -0.0790,  0.0751,  0.0219,  0.0353, -0.0679,\n",
       "                      -0.0426, -0.0056,  0.0060, -0.0590, -0.0701,  0.0734,  0.0390, -0.0363,\n",
       "                       0.0575, -0.0841,  0.0109, -0.0532,  0.0708, -0.0722,  0.0312, -0.0088,\n",
       "                       0.0173, -0.0794,  0.0709, -0.0112,  0.0224, -0.0829, -0.0869,  0.0734,\n",
       "                      -0.0415, -0.0650,  0.0534, -0.0552, -0.0289, -0.0275, -0.0432,  0.0180,\n",
       "                      -0.0226, -0.0281,  0.0002, -0.0199,  0.1229, -0.0557, -0.0184, -0.0205,\n",
       "                       0.0175, -0.0148,  0.0432, -0.0655, -0.0495,  0.0531,  0.0144, -0.0328,\n",
       "                       0.0010,  0.0033, -0.0686,  0.0016, -0.0085, -0.0464, -0.0233,  0.0094,\n",
       "                       0.0225,  0.0391,  0.0209, -0.0588,  0.1686,  0.0427, -0.0446, -0.0273,\n",
       "                      -0.0051,  0.0137, -0.0127, -0.0371, -0.0139,  0.0255, -0.0730,  0.0526,\n",
       "                      -0.0124, -0.0015,  0.0120,  0.0364, -0.0025,  0.0111, -0.0348,  0.0619,\n",
       "                      -0.0047,  0.0009,  0.0744,  0.0769, -0.0424,  0.0320,  0.0835,  0.0165,\n",
       "                      -0.0096, -0.0768,  0.1007,  0.0196,  0.0486,  0.0047,  0.0323, -0.0180,\n",
       "                      -0.0674,  0.0094, -0.0451, -0.0005, -0.0685, -0.0142,  0.0203, -0.0507,\n",
       "                       0.0272, -0.0236, -0.0578, -0.0011,  0.0411,  0.0001, -0.0322,  0.0086,\n",
       "                      -0.0214,  0.0647, -0.0791, -0.0057,  0.0177, -0.0200,  0.0299,  0.0646,\n",
       "                      -0.0658, -0.1006, -0.0661, -0.0689, -0.0037, -0.0624, -0.0460,  0.0895,\n",
       "                       0.0341,  0.0530,  0.0204, -0.0775,  0.0110, -0.0216,  0.0385, -0.0366,\n",
       "                      -0.0579,  0.0405, -0.0102, -0.0565, -0.0043,  0.0147,  0.0456, -0.0404,\n",
       "                      -0.0187,  0.0219, -0.0736, -0.0859, -0.0030,  0.0643, -0.0123, -0.0102,\n",
       "                      -0.0743,  0.0890, -0.0308, -0.0264,  0.0577, -0.0203, -0.0083, -0.0651,\n",
       "                       0.0451, -0.0435,  0.0292, -0.0261,  0.0354,  0.0035, -0.0277, -0.0196,\n",
       "                       0.0182, -0.0858, -0.0839, -0.0546,  0.0629,  0.0604, -0.0218,  0.0147,\n",
       "                      -0.0015, -0.0698, -0.0494,  0.0028,  0.0522,  0.0335, -0.0115,  0.0753,\n",
       "                      -0.0619,  0.0422,  0.0044,  0.0454, -0.0684,  0.0179, -0.0226, -0.0030,\n",
       "                      -0.0617, -0.0637, -0.0652, -0.0495, -0.0714, -0.0535, -0.0157, -0.0381,\n",
       "                       0.0120,  0.0143,  0.0345,  0.0289, -0.0285, -0.0620, -0.0216, -0.0257,\n",
       "                       0.0385, -0.0496,  0.0579,  0.0466, -0.0571, -0.0599, -0.0193, -0.0266,\n",
       "                       0.0049, -0.0357,  0.0638, -0.0580,  0.0538,  0.0760, -0.0033,  0.0680,\n",
       "                      -0.0235,  0.0911,  0.0738,  0.0006,  0.0399,  0.0209, -0.0511, -0.0435,\n",
       "                      -0.0195, -0.0323,  0.0044,  0.0402, -0.0878, -0.0298,  0.0176, -0.0508,\n",
       "                       0.0476, -0.0115, -0.0842,  0.0527,  0.0390, -0.0158,  0.0165, -0.0361,\n",
       "                       0.0085, -0.0452,  0.0153, -0.0071, -0.0498, -0.0760,  0.0348,  0.0085,\n",
       "                      -0.0064,  0.0712, -0.0506,  0.0624,  0.0332, -0.0714,  0.0537, -0.0772,\n",
       "                       0.0541,  0.0785, -0.0391,  0.0693,  0.0470, -0.0116, -0.0311,  0.0331,\n",
       "                      -0.0691,  0.0857,  0.0352,  0.0425,  0.0634, -0.0551, -0.0160, -0.0387,\n",
       "                      -0.0188, -0.0502, -0.0060, -0.0747,  0.0513, -0.0776, -0.0568,  0.0214,\n",
       "                      -0.0533, -0.0672,  0.0358,  0.0017, -0.0148, -0.0578, -0.0461,  0.0368,\n",
       "                       0.0045,  0.0129, -0.0007,  0.0161,  0.0522, -0.0481, -0.0348,  0.0157,\n",
       "                      -0.0520, -0.0341,  0.0069,  0.0516,  0.0183, -0.0617,  0.0350,  0.0425,\n",
       "                      -0.0632,  0.0024, -0.0844, -0.0378, -0.0445, -0.0026, -0.0132,  0.0342,\n",
       "                      -0.0846,  0.0646,  0.0084,  0.0249,  0.0321, -0.0740,  0.0364, -0.0359,\n",
       "                       0.0329,  0.0402, -0.0887, -0.0686,  0.0432,  0.0432,  0.0631, -0.0465,\n",
       "                       0.1112, -0.0594,  0.0460,  0.0355, -0.0241,  0.0615, -0.0040,  0.0333,\n",
       "                       0.0114, -0.0071, -0.0554, -0.0494, -0.0297,  0.0420,  0.0205,  0.0137])),\n",
       "             ('classifier.1.weight',\n",
       "              tensor([[ 0.0179,  0.0376,  0.0012,  ...,  0.0069, -0.0385,  0.0233],\n",
       "                      [ 0.0209, -0.0238, -0.0098,  ..., -0.0318, -0.0299, -0.0058],\n",
       "                      [ 0.0288, -0.0073, -0.0116,  ..., -0.0065, -0.0031,  0.0196],\n",
       "                      ...,\n",
       "                      [ 0.0213,  0.0250,  0.0017,  ...,  0.0251,  0.0011,  0.0253],\n",
       "                      [-0.0154,  0.0251,  0.0288,  ...,  0.0115,  0.0322, -0.0178],\n",
       "                      [ 0.0184,  0.0015, -0.0114,  ..., -0.0401,  0.0225,  0.0283]])),\n",
       "             ('classifier.1.bias',\n",
       "              tensor([-0.0056, -0.0135,  0.0493,  0.0103,  0.0238, -0.0169,  0.0256,  0.0062,\n",
       "                      -0.0534,  0.1221, -0.0450, -0.0685,  0.0203, -0.0748,  0.1189,  0.0141,\n",
       "                      -0.0107, -0.0080, -0.0301, -0.0841, -0.0277, -0.0210, -0.0219,  0.0047,\n",
       "                      -0.0631, -0.0124, -0.0928,  0.0359,  0.0689, -0.0288, -0.0155, -0.0414,\n",
       "                      -0.0052, -0.0878, -0.0233, -0.0044, -0.0201, -0.0195, -0.0422,  0.0517,\n",
       "                       0.0253, -0.0212, -0.0286,  0.0418,  0.0210, -0.0180,  0.0003, -0.0078,\n",
       "                       0.0048, -0.0448,  0.0219,  0.0071, -0.0014, -0.0201, -0.0605,  0.0025,\n",
       "                      -0.0035,  0.0073, -0.0229, -0.0373, -0.0293, -0.0033,  0.0295, -0.0167,\n",
       "                      -0.0086, -0.0221, -0.0191, -0.0453,  0.0341,  0.0511, -0.0158,  0.0203,\n",
       "                      -0.0340,  0.0157, -0.0471, -0.0816, -0.0129, -0.0305,  0.0087, -0.0541,\n",
       "                      -0.0292, -0.0103,  0.0041,  0.0116, -0.0245, -0.0191,  0.0161, -0.0138,\n",
       "                      -0.1980,  0.0309,  0.0193,  0.0020,  0.0161,  0.0404,  0.0080, -0.0343,\n",
       "                       0.0062, -0.0097, -0.0041,  0.0120])),\n",
       "             ('classifier.4.weight',\n",
       "              tensor([[-0.0147, -0.1684, -0.0735, -0.1081, -0.0907,  0.1267, -0.0530, -0.0936,\n",
       "                        0.1312, -0.0771,  0.0839,  0.1023, -0.0739,  0.0751, -0.0840, -0.0279,\n",
       "                        0.0626,  0.0597,  0.0586,  0.1210, -0.0089,  0.1694,  0.1214,  0.0237,\n",
       "                        0.0946, -0.0637,  0.0214, -0.1087, -0.1272,  0.0543, -0.0193,  0.1066,\n",
       "                       -0.0543,  0.1652,  0.0865, -0.0122,  0.0529, -0.0345, -0.0076, -0.0764,\n",
       "                        0.0278,  0.0573,  0.0502, -0.0800, -0.1137,  0.1193, -0.0355,  0.0151,\n",
       "                       -0.1798,  0.0466,  0.0434,  0.0018,  0.0214,  0.0011,  0.0129,  0.0307,\n",
       "                       -0.0498, -0.0091, -0.0035, -0.0112, -0.0267, -0.0794, -0.0501,  0.0693,\n",
       "                        0.0883,  0.0620,  0.0226,  0.1519,  0.0734, -0.0502, -0.0421, -0.1805,\n",
       "                        0.0371, -0.0277,  0.0198,  0.1911, -0.0143, -0.0510, -0.0370,  0.0143,\n",
       "                        0.1460,  0.0843,  0.0018, -0.0342,  0.0163, -0.0657, -0.1054,  0.0172,\n",
       "                        0.2192, -0.1281, -0.1305, -0.0154, -0.1268, -0.1408, -0.0094,  0.1263,\n",
       "                       -0.0601, -0.0976,  0.0341, -0.0480],\n",
       "                      [-0.0374,  0.1194, -0.0383,  0.0110, -0.0439,  0.0397,  0.0676, -0.0331,\n",
       "                       -0.0274,  0.1733, -0.0604, -0.0530,  0.0171, -0.1345,  0.1289, -0.0800,\n",
       "                        0.0408, -0.0355, -0.0469, -0.1091, -0.1011, -0.1761, -0.0562, -0.0396,\n",
       "                       -0.1329, -0.0665, -0.1570, -0.0436,  0.0491, -0.0318, -0.0112, -0.0029,\n",
       "                       -0.0908, -0.1228,  0.0387, -0.0130, -0.0736, -0.0768, -0.0840,  0.0336,\n",
       "                        0.1073, -0.0538, -0.0836,  0.0812,  0.0739, -0.0650, -0.0066, -0.0408,\n",
       "                        0.0938, -0.0600, -0.0603, -0.0511, -0.0621,  0.0371, -0.0950,  0.0145,\n",
       "                       -0.0922,  0.0198, -0.0289,  0.1727, -0.1260,  0.0160,  0.1060,  0.0021,\n",
       "                        0.0626, -0.1107, -0.0868, -0.0943, -0.0646,  0.0598,  0.0008,  0.1455,\n",
       "                       -0.0086, -0.0321, -0.0640, -0.2397,  0.0652, -0.0986, -0.0658, -0.1312,\n",
       "                       -0.1211,  0.0137, -0.0839, -0.0024, -0.0704,  0.0017,  0.1288,  0.1474,\n",
       "                       -0.2386,  0.0631,  0.0983,  0.0850,  0.0669,  0.1161,  0.0229, -0.1024,\n",
       "                       -0.0812, -0.0326,  0.1122,  0.0094]])),\n",
       "             ('classifier.4.bias', tensor([ 0.0333, -0.1350]))])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loaded\n",
    "siamese_en.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(4101, 300)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "siamese_en.encoder.embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizer: sgd\n",
      "Learning rate: 0.01\n",
      "Loading checkpoint: ckpt/siamese-transfer-baseline.pt\n"
     ]
    }
   ],
   "source": [
    "# loss func\n",
    "loss_weights = Variable(torch.FloatTensor([1, 3]))\n",
    "if torch.cuda.is_available():\n",
    "    loss_weights = loss_weights.cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss(loss_weights)\n",
    "\n",
    "# optimizer\n",
    "learning_rate = config['training']['learning_rate']\n",
    "if config['training']['optimizer'] == 'sgd':\n",
    "    optimizer = torch.optim.SGD(filter(lambda x: x.requires_grad, siamese_en.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'adam':\n",
    "    optimizer = torch.optim.Adam(filter(lambda x: x.requires_grad, siamese_en.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'adadelta':\n",
    "    optimizer = torch.optim.Adadelta(filter(lambda x: x.requires_grad, siamese_en.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'rmsprop':\n",
    "    optimizer = torch.optim.RMSprop(filter(lambda x: x.requires_grad, siamese_en.parameters()), lr=learning_rate)\n",
    "print('Optimizer:', config['training']['optimizer'])\n",
    "print('Learning rate:', config['training']['learning_rate'])\n",
    "\n",
    "# log info\n",
    "train_log_string = '%s :: Epoch %i :: Iter %i / %i :: train loss: %0.4f'\n",
    "valid_log_string = '%s :: Epoch %i :: valid loss: %0.4f\\n'\n",
    "\n",
    "# Restore saved model (if one exists).\n",
    "ckpt_path = os.path.join(config['ckpt_dir'], config['experiment_name']+'.pt')\n",
    "if os.path.exists(ckpt_path):\n",
    "    print('Loading checkpoint: %s' % ckpt_path)\n",
    "    ckpt = torch.load(ckpt_path)\n",
    "    epoch = ckpt['epoch']\n",
    "    siamese_en.load_state_dict(ckpt['siamese'])\n",
    "    optimizer.load_state_dict(ckpt['optimizer'])\n",
    "else:\n",
    "    epoch = 1\n",
    "    print('Fresh start!\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train English Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment: siamese-transfer-baseline\n",
      "\n",
      "Start Epoch 1 Training...\n",
      "2018-07-28 15:40:36.333335 :: Epoch 1 :: Iter 5000 / 17029 :: train loss: 0.5641\n",
      "2018-07-28 15:41:53.099676 :: Epoch 1 :: Iter 10000 / 17029 :: train loss: 0.5486\n",
      "2018-07-28 15:43:08.376094 :: Epoch 1 :: Iter 15000 / 17029 :: train loss: 0.5131\n",
      "Train Loss at epoch 1: 0.5361940264701843\n",
      "\n",
      "Epoch 1 Validating...\n",
      "2018-07-28 15:44:05.876684 :: Epoch 1 :: valid loss: 0.4803\n",
      "\n",
      "Model saved!\n",
      "\n",
      "Start Epoch 2 Training...\n",
      "2018-07-28 15:45:18.723418 :: Epoch 2 :: Iter 5000 / 17029 :: train loss: 0.4701\n",
      "2018-07-28 15:46:35.313879 :: Epoch 2 :: Iter 10000 / 17029 :: train loss: 0.4556\n",
      "2018-07-28 15:47:52.252918 :: Epoch 2 :: Iter 15000 / 17029 :: train loss: 0.4445\n",
      "Train Loss at epoch 2: 0.45427393913269043\n",
      "\n",
      "Epoch 2 Validating...\n",
      "2018-07-28 15:48:47.985039 :: Epoch 2 :: valid loss: 0.4464\n",
      "\n",
      "Model saved!\n",
      "\n",
      "Start Epoch 3 Training...\n",
      "2018-07-28 15:50:01.436991 :: Epoch 3 :: Iter 5000 / 17029 :: train loss: 0.4134\n",
      "2018-07-28 15:51:14.784408 :: Epoch 3 :: Iter 10000 / 17029 :: train loss: 0.3980\n",
      "2018-07-28 15:52:28.810164 :: Epoch 3 :: Iter 15000 / 17029 :: train loss: 0.3952\n",
      "Train Loss at epoch 3: 0.3981221914291382\n",
      "\n",
      "Epoch 3 Validating...\n",
      "2018-07-28 15:53:23.882070 :: Epoch 3 :: valid loss: 0.4221\n",
      "\n",
      "Early Stopping!\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Train \"\"\"\n",
    "\n",
    "if config['task'] == 'train':\n",
    "\n",
    "    # save every epoch for visualization\n",
    "    train_loss_record = []\n",
    "    valid_loss_record = []\n",
    "    best_record = 10.0\n",
    "\n",
    "    # training\n",
    "    print('Experiment: {}\\n'.format(config['experiment_name']))\n",
    "\n",
    "    while epoch < config['training']['num_epochs']:\n",
    "\n",
    "        print('Start Epoch {} Training...'.format(epoch))\n",
    "\n",
    "        # loss\n",
    "        train_loss = []\n",
    "        train_loss_sum = []\n",
    "        # dataloader\n",
    "        train_dataloader = DataLoader(dataset=en_trainDS, shuffle=True, num_workers=2, batch_size=1)\n",
    "\n",
    "        for idx, data in enumerate(train_dataloader, 0):\n",
    "\n",
    "            # get data\n",
    "            s1, s2, label = data\n",
    "\n",
    "            # clear gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # input\n",
    "            output = siamese_en(s1, s2)\n",
    "            output = output.squeeze(0)\n",
    "\n",
    "            # label cuda\n",
    "            label = Variable(label)\n",
    "            if torch.cuda.is_available():\n",
    "                label = label.cuda()\n",
    "\n",
    "            # loss backward\n",
    "            loss = criterion(output, label)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_loss.append(loss.data.cpu())\n",
    "            train_loss_sum.append(loss.data.cpu())\n",
    "\n",
    "            # Every once and a while check on the loss\n",
    "            if ((idx + 1) % 5000) == 0:\n",
    "                print(train_log_string % (datetime.now(), epoch, idx + 1, len(en_train), np.mean(train_loss)))\n",
    "                train_loss = []\n",
    "\n",
    "        # Record at every epoch\n",
    "        print('Train Loss at epoch {}: {}\\n'.format(epoch, np.mean(train_loss_sum)))\n",
    "        train_loss_record.append(np.mean(train_loss_sum))\n",
    "\n",
    "        # Valid\n",
    "        print('Epoch {} Validating...'.format(epoch))\n",
    "\n",
    "        # loss\n",
    "        valid_loss = []\n",
    "        # dataloader\n",
    "        valid_dataloader = DataLoader(dataset=en_validDS, shuffle=True, num_workers=2, batch_size=1)\n",
    "\n",
    "        for idx, data in enumerate(valid_dataloader, 0):\n",
    "            # get data\n",
    "            s1, s2, label = data\n",
    "\n",
    "            # input\n",
    "            output = siamese_en(s1, s2)\n",
    "            output = output.squeeze(0)\n",
    "\n",
    "            # label cuda\n",
    "            label = Variable(label)\n",
    "            if torch.cuda.is_available():\n",
    "                label = label.cuda()\n",
    "\n",
    "            # loss\n",
    "            loss = criterion(output, label)\n",
    "            valid_loss.append(loss.data.cpu())\n",
    "\n",
    "        print(valid_log_string % (datetime.now(), epoch, np.mean(valid_loss)))\n",
    "        # Record\n",
    "        valid_loss_record.append(np.mean(valid_loss))\n",
    "        epoch += 1\n",
    "\n",
    "        if np.mean(valid_loss)-np.mean(train_loss_sum) > 0.02:\n",
    "             print(\"Early Stopping!\")\n",
    "             break\n",
    "\n",
    "        # Keep track of best record\n",
    "        if np.mean(valid_loss) < best_record:\n",
    "            best_record = np.mean(valid_loss)\n",
    "            # save the best model\n",
    "            state_dict = {\n",
    "                'epoch': epoch,\n",
    "                'siamese': siamese_en.state_dict(),\n",
    "                'optimizer': optimizer.state_dict(),\n",
    "            }\n",
    "            torch.save(state_dict, ckpt_path)\n",
    "            print('Model saved!\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfer weights to Train Spanish Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ckpt_path = 'ckpt/siamese-transfer-baseline.pt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "config['experiment_name'] = config['experiment_name'] + '-Spain'\n",
    "config['embedding_matrix'] = sp_embedding\n",
    "siamese_sp = Siamese_lstm(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transfering English Model from: ckpt/siamese-transfer-baseline.pt\n"
     ]
    }
   ],
   "source": [
    "# Restore saved English model \n",
    "\n",
    "print('Transfering English Model from: %s' % ckpt_path)\n",
    "ckpt = torch.load(ckpt_path)\n",
    "en_epoch = ckpt['epoch']\n",
    "epoch = 1\n",
    "siamese_sp.load_state_dict(ckpt['siamese'])\n",
    "\n",
    "# Another path to save sp model\n",
    "ckpt_path = os.path.join(config['ckpt_dir'], config['experiment_name']+'.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Siamese_lstm(\n",
      "  (encoder): LSTMEncoder(\n",
      "    (embedding): Embedding(4101, 300)\n",
      "    (lstm): LSTM(300, 150, dropout=0.5)\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.1)\n",
      "    (1): Linear(in_features=600, out_features=100, bias=True)\n",
      "    (2): Tanh()\n",
      "    (3): Dropout(p=0.1)\n",
      "    (4): Linear(in_features=100, out_features=2, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(siamese_sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['training']['learning_rate'] = 0.001\n",
    "config['training']['optimizer'] = 'adadelta'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizer: adadelta\n",
      "Learning rate: 0.001\n"
     ]
    }
   ],
   "source": [
    "# loss func\n",
    "loss_weights = Variable(torch.FloatTensor([1, 3]))\n",
    "if torch.cuda.is_available():\n",
    "    loss_weights = loss_weights.cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss(loss_weights)\n",
    "\n",
    "# optimizer\n",
    "learning_rate = config['training']['learning_rate']\n",
    "if config['training']['optimizer'] == 'sgd':\n",
    "    optimizer = torch.optim.SGD(filter(lambda x: x.requires_grad, siamese_sp.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'adam':\n",
    "    optimizer = torch.optim.Adam(filter(lambda x: x.requires_grad, siamese_sp.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'adadelta':\n",
    "    optimizer = torch.optim.Adadelta(filter(lambda x: x.requires_grad, siamese_sp.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'rmsprop':\n",
    "    optimizer = torch.optim.RMSprop(filter(lambda x: x.requires_grad, siamese_sp.parameters()), lr=learning_rate)\n",
    "print('Optimizer:', config['training']['optimizer'])\n",
    "print('Learning rate:', config['training']['learning_rate'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 1\n",
    "# log info\n",
    "train_log_string = '%s :: Epoch %i :: Iter %i / %i :: train loss: %0.4f'\n",
    "valid_log_string = '%s :: Epoch %i :: valid loss: %0.4f\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[tensor([1]), tensor([9]), tensor([1362]), tensor([2])] \n",
      " [tensor([1]), tensor([4]), tensor([5]), tensor([40]), tensor([9]), tensor([21]), tensor([554]), tensor([2])]\n",
      "out tensor([[ 0.6446, -0.6029]], grad_fn=<SqueezeBackward1>)\n",
      "tensor(0.2525, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "train_dataloader = DataLoader(dataset=sp_trainDS, shuffle=True, batch_size=1)\n",
    "\n",
    "for idx, data in enumerate(train_dataloader, 0):\n",
    "    s1, s2, label = data\n",
    "    print(s1,'\\n',s2)\n",
    "    output = siamese_sp(s1, s2)\n",
    "    output = output.squeeze(0)\n",
    "    print('out',output)\n",
    "    \n",
    "    # clear gradients\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    # loss backward\n",
    "    loss = criterion(output, label)\n",
    "    print(loss)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment: siamese-transfer-baseline-Spain\n",
      "\n",
      "Start Epoch 1 Training...\n",
      "2018-07-28 17:31:06.542327 :: Epoch 1 :: Iter 5000 / 17063 :: train loss: 0.5542\n",
      "2018-07-28 17:32:40.870212 :: Epoch 1 :: Iter 10000 / 17063 :: train loss: 0.5594\n",
      "2018-07-28 17:34:16.170620 :: Epoch 1 :: Iter 15000 / 17063 :: train loss: 0.5645\n",
      "Train Loss at epoch 1: 0.5586052536964417\n",
      "\n",
      "Epoch 1 Validating...\n",
      "2018-07-28 17:35:22.929568 :: Epoch 1 :: valid loss: 0.5712\n",
      "\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'best_record' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-77cce7687151>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0;31m# Keep track of best record\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_loss\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbest_record\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m             \u001b[0mbest_record\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0;31m# save the best model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'best_record' is not defined"
     ]
    }
   ],
   "source": [
    "\"\"\" Train \"\"\"\n",
    "\n",
    "if config['task'] == 'train':\n",
    "\n",
    "    # save every epoch for visualization\n",
    "    train_loss_record = []\n",
    "    valid_loss_record = []\n",
    "#     best_record = 10.0\n",
    "\n",
    "    # training\n",
    "    print('Experiment: {}\\n'.format(config['experiment_name']))\n",
    "\n",
    "    while epoch < config['training']['num_epochs']:\n",
    "\n",
    "        print('Start Epoch {} Training...'.format(epoch))\n",
    "\n",
    "        # loss\n",
    "        train_loss = []\n",
    "        train_loss_sum = []\n",
    "        # dataloader\n",
    "        train_dataloader = DataLoader(dataset=sp_trainDS, shuffle=True, num_workers=2, batch_size=1)\n",
    "\n",
    "        for idx, data in enumerate(train_dataloader, 0):\n",
    "\n",
    "            # get data\n",
    "            s1, s2, label = data\n",
    "\n",
    "            # clear gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # input\n",
    "            output = siamese_sp(s1, s2)\n",
    "            output = output.squeeze(0)\n",
    "\n",
    "            # label cuda\n",
    "            label = Variable(label)\n",
    "            if torch.cuda.is_available():\n",
    "                label = label.cuda()\n",
    "\n",
    "            # loss backward\n",
    "            loss = criterion(output, label)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_loss.append(loss.data.cpu())\n",
    "            train_loss_sum.append(loss.data.cpu())\n",
    "\n",
    "            # Every once and a while check on the loss\n",
    "            if ((idx + 1) % 5000) == 0:\n",
    "                print(train_log_string % (datetime.now(), epoch, idx + 1, len(sp_train), np.mean(train_loss)))\n",
    "                train_loss = []\n",
    "\n",
    "        # Record at every epoch\n",
    "        print('Train Loss at epoch {}: {}\\n'.format(epoch, np.mean(train_loss_sum)))\n",
    "        train_loss_record.append(np.mean(train_loss_sum))\n",
    "\n",
    "        # Valid\n",
    "        print('Epoch {} Validating...'.format(epoch))\n",
    "\n",
    "        # loss\n",
    "        valid_loss = []\n",
    "        # dataloader\n",
    "        valid_dataloader = DataLoader(dataset=sp_validDS, shuffle=True, num_workers=2, batch_size=1)\n",
    "\n",
    "        for idx, data in enumerate(valid_dataloader, 0):\n",
    "            # get data\n",
    "            s1, s2, label = data\n",
    "\n",
    "            # input\n",
    "            output = siamese_sp(s1, s2)\n",
    "            output = output.squeeze(0)\n",
    "\n",
    "            # label cuda\n",
    "            label = Variable(label)\n",
    "            if torch.cuda.is_available():\n",
    "                label = label.cuda()\n",
    "\n",
    "            # loss\n",
    "            loss = criterion(output, label)\n",
    "            valid_loss.append(loss.data.cpu())\n",
    "\n",
    "        print(valid_log_string % (datetime.now(), epoch, np.mean(valid_loss)))\n",
    "        # Record\n",
    "        valid_loss_record.append(np.mean(valid_loss))\n",
    "        epoch += 1\n",
    "\n",
    "        if np.mean(valid_loss)-np.mean(train_loss_sum) > 0.02:\n",
    "             print(\"Early Stopping!\")\n",
    "             break\n",
    "\n",
    "        # Keep track of best record\n",
    "        if np.mean(valid_loss) < best_record:\n",
    "            best_record = np.mean(valid_loss)\n",
    "            # save the best model\n",
    "            state_dict = {\n",
    "                'epoch': epoch,\n",
    "                'siamese': siamese_sp.state_dict(),\n",
    "                'optimizer': optimizer.state_dict(),\n",
    "            }\n",
    "            torch.save(state_dict, ckpt_path)\n",
    "            print('Model saved!\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}