ner_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fdc06a12330>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from collections import OrderedDict\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import copy\n",
    "torch.manual_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Parse():\n",
    "    def __init__(self):\n",
    "        self.dataset = 'conll'\n",
    "        self.result_path = 'neural_ner/results'\n",
    "        self.usemodel = 'CNN_BiLSTM_CRF'\n",
    "        self.worddim = 100\n",
    "        self.pretrnd = 'wordvectors/glove.6B.100d.txt'\n",
    "        self.reload = 0\n",
    "        self.checkpoint = '.'\n",
    "        self.num_epochs = 10\n",
    "\n",
    "opt=Parse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.autograd.Variable(torch.randn(4)).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-0.2934\n",
       "[torch.cuda.FloatTensor of size 1 (GPU 0)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import codecs\n",
    "import random\n",
    "random.seed(0)\n",
    "\n",
    "START_TAG = '<START>'\n",
    "STOP_TAG = '<STOP>'\n",
    "\n",
    "def get_name(parameters):\n",
    "    \"\"\"\n",
    "    Generate a model name from its parameters.\n",
    "    \"\"\"\n",
    "    l = []\n",
    "    for k, v in parameters.items():\n",
    "        if type(v) is str and \"/\" in v:\n",
    "            l.append((k, v[::-1][:v[::-1].index('/')][::-1]))\n",
    "        else:\n",
    "            l.append((k, v))\n",
    "    name = \",\".join([\"%s=%s\" % (k, str(v).replace(',', '')) for k, v in l])\n",
    "    return \"\".join(i for i in name if i not in \"\\/:*?<>|\")\n",
    "\n",
    "\n",
    "def set_values(name, param, pretrained):\n",
    "    \"\"\"\n",
    "    Initialize a network parameter with pretrained values.\n",
    "    We check that sizes are compatible.\n",
    "    \"\"\"\n",
    "    param_value = param.get_value()\n",
    "    if pretrained.size != param_value.size:\n",
    "        raise Exception(\n",
    "            \"Size mismatch for parameter %s. Expected %i, found %i.\"\n",
    "            % (name, param_value.size, pretrained.size)\n",
    "        )\n",
    "    param.set_value(np.reshape(\n",
    "        pretrained, param_value.shape).astype(np.float32))\n",
    "\n",
    "\n",
    "def create_dico(item_list):\n",
    "    \"\"\"\n",
    "    Create a dictionary of items from a list of list of items.\n",
    "    \"\"\"\n",
    "    assert type(item_list) is list\n",
    "    dico = {}\n",
    "    for items in item_list:\n",
    "        for item in items:\n",
    "            if item not in dico:\n",
    "                dico[item] = 1\n",
    "            else:\n",
    "                dico[item] += 1\n",
    "    return dico\n",
    "\n",
    "\n",
    "def create_mapping(dico):\n",
    "    \"\"\"\n",
    "    Create a mapping (item to ID / ID to item) from a dictionary.\n",
    "    Items are ordered by decreasing frequency.\n",
    "    \"\"\"\n",
    "    sorted_items = sorted(dico.items(), key=lambda x: (-x[1], x[0]))\n",
    "    id_to_item = {i: v[0] for i, v in enumerate(sorted_items)}\n",
    "    item_to_id = {v: k for k, v in id_to_item.items()}\n",
    "    return item_to_id, id_to_item\n",
    "\n",
    "\n",
    "def zero_digits(s):\n",
    "    \"\"\"\n",
    "    Replace every digit in a string by a zero.\n",
    "    \"\"\"\n",
    "    return re.sub('\\d', '0', s)\n",
    "\n",
    "\n",
    "def iob2(tags):\n",
    "    \"\"\"\n",
    "    Check that tags have a valid IOB format.\n",
    "    Tags in IOB1 format are converted to IOB2.\n",
    "    \"\"\"\n",
    "    for i, tag in enumerate(tags):\n",
    "        if tag == 'O':\n",
    "            continue\n",
    "        split = tag.split('-')\n",
    "        if len(split) != 2 or split[0] not in ['I', 'B']:\n",
    "            return False\n",
    "        if split[0] == 'B':\n",
    "            continue\n",
    "        elif i == 0 or tags[i - 1] == 'O':  # conversion IOB1 to IOB2\n",
    "            tags[i] = 'B' + tag[1:]\n",
    "        elif tags[i - 1][1:] == tag[1:]:\n",
    "            continue\n",
    "        else:  # conversion IOB1 to IOB2\n",
    "            tags[i] = 'B' + tag[1:]\n",
    "    return True\n",
    "\n",
    "\n",
    "def iob_iobes(tags):\n",
    "    \"\"\"\n",
    "    IOB -> IOBES\n",
    "    \"\"\"\n",
    "    new_tags = []\n",
    "    for i, tag in enumerate(tags):\n",
    "        if tag == 'O':\n",
    "            new_tags.append(tag)\n",
    "        elif tag.split('-')[0] == 'B':\n",
    "            if i + 1 != len(tags) and \\\n",
    "               tags[i + 1].split('-')[0] == 'I':\n",
    "                new_tags.append(tag)\n",
    "            else:\n",
    "                new_tags.append(tag.replace('B-', 'S-'))\n",
    "        elif tag.split('-')[0] == 'I':\n",
    "            if i + 1 < len(tags) and \\\n",
    "                    tags[i + 1].split('-')[0] == 'I':\n",
    "                new_tags.append(tag)\n",
    "            else:\n",
    "                new_tags.append(tag.replace('I-', 'E-'))\n",
    "        else:\n",
    "            raise Exception('Invalid IOB format!')\n",
    "    return new_tags\n",
    "\n",
    "\n",
    "def iobes_iob(tags):\n",
    "    \"\"\"\n",
    "    IOBES -> IOB\n",
    "    \"\"\"\n",
    "    new_tags = []\n",
    "    for i, tag in enumerate(tags):\n",
    "        if tag.split('-')[0] == 'B':\n",
    "            new_tags.append(tag)\n",
    "        elif tag.split('-')[0] == 'I':\n",
    "            new_tags.append(tag)\n",
    "        elif tag.split('-')[0] == 'S':\n",
    "            new_tags.append(tag.replace('S-', 'B-'))\n",
    "        elif tag.split('-')[0] == 'E':\n",
    "            new_tags.append(tag.replace('E-', 'I-'))\n",
    "        elif tag.split('-')[0] == 'O':\n",
    "            new_tags.append(tag)\n",
    "        else:\n",
    "            raise Exception('Invalid format!')\n",
    "    return new_tags\n",
    "\n",
    "\n",
    "def insert_singletons(words, singletons, p=0.5):\n",
    "    \"\"\"\n",
    "    Replace singletons by the unknown word with a probability p.\n",
    "    \"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        if word in singletons and np.random.uniform() < p:\n",
    "            new_words.append(0)\n",
    "        else:\n",
    "            new_words.append(word)\n",
    "    return new_words\n",
    "\n",
    "\n",
    "def pad_word_chars(words):\n",
    "    \"\"\"\n",
    "    Pad the characters of the words in a sentence.\n",
    "    Input:\n",
    "        - list of lists of ints (list of words, a word being a list of char indexes)\n",
    "    Output:\n",
    "        - padded list of lists of ints\n",
    "        - padded list of lists of ints (where chars are reversed)\n",
    "        - list of ints corresponding to the index of the last character of each word\n",
    "    \"\"\"\n",
    "    max_length = max([len(word) for word in words])\n",
    "    char_for = []\n",
    "    char_rev = []\n",
    "    char_pos = []\n",
    "    for word in words:\n",
    "        padding = [0] * (max_length - len(word))\n",
    "        char_for.append(word + padding)\n",
    "        char_rev.append(word[::-1] + padding)\n",
    "        char_pos.append(len(word) - 1)\n",
    "    return char_for, char_rev, char_pos\n",
    "\n",
    "\n",
    "def create_input(data, parameters, add_label, singletons=None):\n",
    "    \"\"\"\n",
    "    Take sentence data and return an input for\n",
    "    the training or the evaluation function.\n",
    "    \"\"\"\n",
    "    words = data['words']\n",
    "    chars = data['chars']\n",
    "    if singletons is not None:\n",
    "        words = insert_singletons(words, singletons)\n",
    "    if parameters['cap_dim']:\n",
    "        caps = data['caps']\n",
    "    char_for, char_rev, char_pos = pad_word_chars(chars)\n",
    "    input = []\n",
    "    if parameters['word_dim']:\n",
    "        input.append(words)\n",
    "    if parameters['char_dim']:\n",
    "        input.append(char_for)\n",
    "        if parameters['char_bidirect']:\n",
    "            input.append(char_rev)\n",
    "        input.append(char_pos)\n",
    "    if parameters['cap_dim']:\n",
    "        input.append(caps)\n",
    "    if add_label:\n",
    "        input.append(data['tags'])\n",
    "    return input\n",
    "\n",
    "def char_mapping(sentences):\n",
    "    \"\"\"\n",
    "    Create a dictionary and mapping of characters, sorted by frequency.\n",
    "    \"\"\"\n",
    "    chars = [\"\".join([w[0] for w in s]) for s in sentences]\n",
    "    dico = create_dico(chars)\n",
    "    dico['<PAD>'] = 10000000\n",
    "    # dico[';'] = 0\n",
    "    char_to_id, id_to_char = create_mapping(dico)\n",
    "    print(\"Found %i unique characters\" % len(dico))\n",
    "    return dico, char_to_id, id_to_char\n",
    "\n",
    "\n",
    "def tag_mapping(sentences):\n",
    "    \"\"\"\n",
    "    Create a dictionary and a mapping of tags, sorted by frequency.\n",
    "    \"\"\"\n",
    "    tags = [[word[-1] for word in s] for s in sentences]\n",
    "    dico = create_dico(tags)\n",
    "    dico[START_TAG] = -1\n",
    "    dico[STOP_TAG] = -2\n",
    "    tag_to_id, id_to_tag = create_mapping(dico)\n",
    "    print(\"Found %i unique named entity tags\" % len(dico))\n",
    "    return dico, tag_to_id, id_to_tag\n",
    "\n",
    "\n",
    "def cap_feature(s):\n",
    "    \"\"\"\n",
    "    Capitalization feature:\n",
    "    0 = low caps\n",
    "    1 = all caps\n",
    "    2 = first letter caps\n",
    "    3 = one capital (not first letter)\n",
    "    \"\"\"\n",
    "    if s.lower() == s:\n",
    "        return 0\n",
    "    elif s.upper() == s:\n",
    "        return 1\n",
    "    elif s[0].upper() == s[0]:\n",
    "        return 2\n",
    "    else:\n",
    "        return 3\n",
    "\n",
    "\n",
    "def prepare_sentence(str_words, word_to_id, char_to_id, lower=False):\n",
    "    \"\"\"\n",
    "    Prepare a sentence for evaluation.\n",
    "    \"\"\"\n",
    "    def f(x): return x.lower() if lower else x\n",
    "    words = [word_to_id[f(w) if f(w) in word_to_id else '<UNK>']\n",
    "             for w in str_words]\n",
    "    chars = [[char_to_id[c] for c in w if c in char_to_id]\n",
    "             for w in str_words]\n",
    "    caps = [cap_feature(w) for w in str_words]\n",
    "    return {\n",
    "        'str_words': str_words,\n",
    "        'words': words,\n",
    "        'chars': chars,\n",
    "        'caps': caps\n",
    "    }\n",
    "\n",
    "\n",
    "def prepare_dataset(sentences, word_to_id, char_to_id, tag_to_id, lower=True):\n",
    "    \"\"\"\n",
    "    Prepare the dataset. Return a list of lists of dictionaries containing:\n",
    "        - word indexes\n",
    "        - word char indexes\n",
    "        - tag indexes\n",
    "    \"\"\"\n",
    "    def f(x): return x.lower() if lower else x\n",
    "    data = []\n",
    "    for s in sentences:\n",
    "        str_words = [w[0] for w in s]\n",
    "        words = [word_to_id[f(w) if f(w) in word_to_id else '<UNK>']\n",
    "                 for w in str_words]\n",
    "        # Skip characters that are not in the training set\n",
    "        chars = [[char_to_id[c] for c in w if c in char_to_id]\n",
    "                 for w in str_words]\n",
    "        caps = [cap_feature(w) for w in str_words]\n",
    "        tags = [tag_to_id[w[-1]] for w in s]\n",
    "        data.append({\n",
    "            'str_words': str_words,\n",
    "            'words': words,\n",
    "            'chars': chars,\n",
    "            'caps': caps,\n",
    "            'tags': tags,\n",
    "        })\n",
    "    return data\n",
    "\n",
    "\n",
    "def augment_with_pretrained(dictionary, ext_emb_path, words):\n",
    "    \"\"\"\n",
    "    Augment the dictionary with words that have a pretrained embedding.\n",
    "    If `words` is None, we add every word that has a pretrained embedding\n",
    "    to the dictionary, otherwise, we only add the words that are given by\n",
    "    `words` (typically the words in the development and test sets.)\n",
    "    \"\"\"\n",
    "    print('Loading pretrained embeddings from %s...' % ext_emb_path)\n",
    "    assert os.path.isfile(ext_emb_path)\n",
    "\n",
    "    # Load pretrained embeddings from file\n",
    "    pretrained = set([\n",
    "        line.rstrip().split()[0].strip()\n",
    "        for line in codecs.open(ext_emb_path, 'r', 'utf-8')\n",
    "        if len(ext_emb_path) > 0\n",
    "    ])\n",
    "    \n",
    "    if words is None:\n",
    "        for word in pretrained:\n",
    "            if word not in dictionary:\n",
    "                dictionary[word] = 0\n",
    "    else:\n",
    "        for word in words:\n",
    "            if any(x in pretrained for x in [\n",
    "                word,\n",
    "                word.lower(),\n",
    "                re.sub('\\d', '0', word.lower())\n",
    "            ]) and word not in dictionary:\n",
    "                dictionary[word] = 0\n",
    "\n",
    "    word_to_id, id_to_word = create_mapping(dictionary)\n",
    "    return dictionary, word_to_id, id_to_word\n",
    "\n",
    "\n",
    "def pad_seq(seq, max_length, PAD_token=0):\n",
    "    \n",
    "    seq += [PAD_token for i in range(max_length - len(seq))]\n",
    "    return seq\n",
    "\n",
    "def log_sum_exp(vec, dim=-1, keepdim = False):\n",
    "    max_score, _ = vec.max(dim, keepdim=keepdim)\n",
    "    if keepdim:\n",
    "        stable_vec = vec - max_score\n",
    "    else:\n",
    "        stable_vec = vec - max_score.unsqueeze(dim)\n",
    "    output = max_score + (stable_vec.exp().sum(dim, keepdim=keepdim)).log()\n",
    "    return output\n",
    "\n",
    "def create_batches(dataset, batch_size, order='keep', str_words=False, tag_padded= True):\n",
    "    \n",
    "        newdata = copy.deepcopy(dataset)\n",
    "        if order=='sort':\n",
    "            newdata.sort(key = lambda x:len(x['words']))\n",
    "        elif order=='random':\n",
    "            random.shuffle(newdata)\n",
    "        \n",
    "        newdata = np.array(newdata)  \n",
    "        batches = []\n",
    "        num_batches = np.ceil(len(dataset)/float(batch_size)).astype('int')\n",
    "        \n",
    "        for i in range(num_batches):\n",
    "            batch_data = newdata[(i*batch_size):min(len(dataset),(i+1)*batch_size)]\n",
    "            \n",
    "            words_seqs = [itm['words'] for itm in batch_data]\n",
    "            caps_seqs = [itm['caps'] for itm in batch_data]\n",
    "            target_seqs = [itm['tags'] for itm in batch_data]\n",
    "            chars_seqs = [itm['chars'] for itm in batch_data]\n",
    "            str_words_seqs = [itm['str_words'] for itm in batch_data]\n",
    "            \n",
    "            seq_pairs = sorted(zip(words_seqs, caps_seqs, target_seqs, chars_seqs, str_words_seqs), \n",
    "                               key=lambda p: len(p[0]), reverse=True)\n",
    "            \n",
    "            words_seqs, caps_seqs, target_seqs, chars_seqs, str_words_seqs = zip(*seq_pairs)\n",
    "            words_lengths = np.array([len(s) for s in words_seqs])\n",
    "            \n",
    "            words_padded = np.array([pad_seq(s, np.max(words_lengths)) for s in words_seqs])\n",
    "            caps_padded = np.array([pad_seq(s, np.max(words_lengths)) for s in caps_seqs])\n",
    "            \n",
    "            if tag_padded:\n",
    "                target_padded = np.array([pad_seq(s, np.max(words_lengths)) for s in target_seqs])\n",
    "            else:\n",
    "                target_padded = target_seqs\n",
    "            \n",
    "            words_mask = (words_padded!=0).astype('int')\n",
    "            \n",
    "            chars_pseqs = [pad_seq(s, max(words_lengths), []) for s in chars_seqs]\n",
    "            chars_lengths = np.array([[len(s) for s in w] for w in chars_pseqs]).reshape(-1)\n",
    "            chars_padded = np.array([[pad_seq(s, np.max(chars_lengths)) \n",
    "                                      for s in w] for w in chars_pseqs]).reshape(-1,np.max(chars_lengths))\n",
    "    \n",
    "            if str_words:\n",
    "                outputdict = {'words':words_padded, 'caps':caps_padded, 'tags': target_padded, \n",
    "                              'chars': chars_padded, 'wordslen': words_lengths, 'charslen': chars_lengths,\n",
    "                              'tagsmask':words_mask, 'str_words': str_words_seqs}\n",
    "            else:\n",
    "                outputdict = {'words':words_padded, 'caps':caps_padded, 'tags': target_padded, \n",
    "                              'chars': chars_padded, 'wordslen': words_lengths, 'charslen': chars_lengths,\n",
    "                              'tagsmask':words_mask}\n",
    "            \n",
    "            batches.append(outputdict)\n",
    "        \n",
    "        return batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torch.nn import init\n",
    "\n",
    "class Initializer(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def init_embedding(self, input_embedding):\n",
    "        bias = np.sqrt(3.0 / input_embedding.size(1))\n",
    "        nn.init.uniform(input_embedding, -bias, bias)\n",
    "    \n",
    "    def init_linear(self, input_linear):\n",
    "        bias = np.sqrt(6.0 / (input_linear.weight.size(0) + input_linear.weight.size(1)))\n",
    "        nn.init.uniform(input_linear.weight, -bias, bias)\n",
    "        if input_linear.bias is not None:\n",
    "            input_linear.bias.data.zero_()\n",
    "    \n",
    "    def init_lstm(self, input_lstm):\n",
    "        for ind in range(0, input_lstm.num_layers):\n",
    "            weight = eval('input_lstm.weight_ih_l' + str(ind))\n",
    "            bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))\n",
    "            nn.init.uniform(weight, -bias, bias)\n",
    "            weight = eval('input_lstm.weight_hh_l' + str(ind))\n",
    "            bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))\n",
    "            nn.init.uniform(weight, -bias, bias)\n",
    "        \n",
    "        if input_lstm.bidirectional:\n",
    "            for ind in range(0, input_lstm.num_layers):\n",
    "                weight = eval('input_lstm.weight_ih_l' + str(ind) + '_reverse')\n",
    "                bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))\n",
    "                nn.init.uniform(weight, -bias, bias)\n",
    "                weight = eval('input_lstm.weight_hh_l' + str(ind) + '_reverse')\n",
    "                bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))\n",
    "                nn.init.uniform(weight, -bias, bias)\n",
    "        \n",
    "        if input_lstm.bias:\n",
    "            \n",
    "            for ind in range(0, input_lstm.num_layers):\n",
    "                weight = eval('input_lstm.bias_ih_l' + str(ind))\n",
    "                weight.data.zero_()\n",
    "                weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1\n",
    "                weight = eval('input_lstm.bias_hh_l' + str(ind))\n",
    "                weight.data.zero_()\n",
    "                weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1\n",
    "            \n",
    "            if input_lstm.bidirectional:\n",
    "                for ind in range(0, input_lstm.num_layers):\n",
    "                    weight = eval('input_lstm.bias_ih_l' + str(ind) + '_reverse')\n",
    "                    weight.data.zero_()\n",
    "                    weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1\n",
    "                    weight = eval('input_lstm.bias_hh_l' + str(ind) + '_reverse')\n",
    "                    weight.data.zero_()\n",
    "                    weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "from torch.autograd import Variable\n",
    "import codecs\n",
    "import cPickle\n",
    "\n",
    "class Loader(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def pad_sequence_cnn(self, chars):\n",
    "        d = {}\n",
    "        chars_length = [len(c) for c in chars]\n",
    "        chars_maxlen = max(chars_length)\n",
    "        chars_mask = np.zeros((len(chars_length), chars_maxlen), dtype='int')\n",
    "        for i, c in enumerate(chars):\n",
    "            chars_mask[i, :chars_length[i]] = c\n",
    "        return chars_mask, chars_length, d\n",
    "    \n",
    "    \n",
    "    def pad_sequence_rnn(self, chars):\n",
    "        chars_sorted = sorted(chars, key=lambda p: len(p), reverse=True)\n",
    "        d = {}\n",
    "        for i, ci in enumerate(chars):\n",
    "            for j, cj in enumerate(chars_sorted):\n",
    "                if ci == cj and not j in d and not i in d.values():\n",
    "                    d[j] = i\n",
    "                    continue\n",
    "        chars_length = [len(c) for c in chars_sorted]\n",
    "        chars_maxlen = max(chars_length)\n",
    "        chars_mask = np.zeros((len(chars_sorted), char_maxlen), dtype='int')\n",
    "        for i, c in enumerate(chars_sorted):\n",
    "            chars_mask[i, :chars_length[i]] = c\n",
    "        return chars_mask, chars_length, d\n",
    "    \n",
    "    def update_tag_scheme(self, sentences, tag_scheme):\n",
    "        \n",
    "        for i, s in enumerate(sentences):\n",
    "            tags = [w[-1] for w in s]\n",
    "            if not iob2(tags):\n",
    "                s_str = '\\n'.join(' '.join(w) for w in s)\n",
    "                raise Exception('Sentences should be given in IOB format! ' +\n",
    "                                'Please check sentence %i:\\n%s' % (i, s_str))\n",
    "            if tag_scheme == 'iob':\n",
    "                for word, new_tag in zip(s, tags):\n",
    "                    word[-1] = new_tag\n",
    "            elif tag_scheme == 'iobes':\n",
    "                new_tags = iob_iobes(tags)\n",
    "                for word, new_tag in zip(s, new_tags):\n",
    "                    word[-1] = new_tag\n",
    "            else:\n",
    "                raise Exception('Unknown tagging scheme!')\n",
    "                \n",
    "    def word_mapping(self, sentences, lower):\n",
    "        \n",
    "        words = [[x[0].lower() if lower else x[0] for x in s] for s in sentences]\n",
    "        dico = create_dico(words)\n",
    "\n",
    "        dico['<PAD>'] = 10000001\n",
    "        dico['<UNK>'] = 10000000\n",
    "        dico = {k:v for k,v in dico.items() if v>=3}\n",
    "        word_to_id, id_to_word = create_mapping(dico)\n",
    "\n",
    "        print(\"Found %i unique words (%i in total)\" % (\n",
    "            len(dico), sum(len(x) for x in words)\n",
    "        ))\n",
    "        return dico, word_to_id, id_to_word\n",
    "    \n",
    "    def load_conll_sentences(self, path, lower, zeros):\n",
    "        \n",
    "        sentences = []\n",
    "        sentence = []\n",
    "        for line in codecs.open(path, 'r', 'utf-8'):\n",
    "            line = zero_digits(line.rstrip()) if zeros else line.rstrip()\n",
    "            if not line:\n",
    "                if len(sentence) > 0:\n",
    "                    if 'DOCSTART' not in sentence[0][0]:\n",
    "                        sentences.append(sentence)\n",
    "                    sentence = []\n",
    "            else:\n",
    "                word = line.split()\n",
    "                assert len(word) >= 2\n",
    "                sentence.append(word)\n",
    "        if len(sentence) > 0:\n",
    "            if 'DOCSTART' not in sentence[0][0]:\n",
    "                sentences.append(sentence)\n",
    "        return sentences\n",
    "    \n",
    "    def load_conll(self, dataset ,parameters):\n",
    "        \n",
    "        zeros = parameters['zeros']\n",
    "        lower = parameters['lower']\n",
    "        word_dim = parameters['wrdim']\n",
    "        pretrained = parameters['ptrnd']\n",
    "        tag_scheme = parameters['tgsch']\n",
    "        \n",
    "        train_path = os.path.join(dataset,'eng.train')\n",
    "        dev_path = os.path.join(dataset,'eng.testa')\n",
    "        test_path = os.path.join(dataset,'eng.testb')\n",
    "        test_train_path = os.path.join(dataset,'eng.train54019')\n",
    "        \n",
    "        train_sentences = self.load_conll_sentences(train_path, lower, zeros)\n",
    "        dev_sentences = self.load_conll_sentences(dev_path, lower, zeros)\n",
    "        test_sentences = self.load_conll_sentences(test_path, lower, zeros)\n",
    "        test_train_sentences = self.load_conll_sentences(test_train_path, lower, zeros)\n",
    "        \n",
    "        self.update_tag_scheme(train_sentences, tag_scheme)\n",
    "        self.update_tag_scheme(dev_sentences, tag_scheme)\n",
    "        self.update_tag_scheme(test_sentences, tag_scheme)\n",
    "        self.update_tag_scheme(test_train_sentences, tag_scheme)\n",
    "        \n",
    "        dico_words_train = self.word_mapping(train_sentences, lower)[0]\n",
    "        \n",
    "        all_embedding = 1\n",
    "        dico_words, word_to_id, id_to_word = augment_with_pretrained(\n",
    "                dico_words_train.copy(),\n",
    "                pretrained,\n",
    "                list(itertools.chain.from_iterable(\n",
    "                    [[w[0] for w in s] for s in dev_sentences + test_sentences])\n",
    "                ) if not all_embedding else None)\n",
    "\n",
    "        dico_chars, char_to_id, id_to_char = char_mapping(train_sentences)\n",
    "        dico_tags, tag_to_id, id_to_tag = tag_mapping(train_sentences)\n",
    "        \n",
    "        train_data = prepare_dataset(train_sentences, word_to_id, char_to_id, tag_to_id, lower)\n",
    "        dev_data = prepare_dataset(dev_sentences, word_to_id, char_to_id, tag_to_id, lower)\n",
    "        test_data = prepare_dataset(test_sentences, word_to_id, char_to_id, tag_to_id, lower)\n",
    "        test_train_data = prepare_dataset(test_train_sentences, word_to_id, char_to_id, tag_to_id, lower)\n",
    "        \n",
    "        print(\"%i / %i / %i sentences in train / dev / test.\" % (\n",
    "              len(train_data), len(dev_data), len(test_data)))\n",
    "        \n",
    "        mapping_file = os.path.join(dataset,'mappinghawa.pkl')\n",
    "        \n",
    "        if not os.path.isfile(mapping_file):\n",
    "            all_word_embeds = {}\n",
    "            for i, line in enumerate(codecs.open(pretrained, 'r', 'utf-8')):\n",
    "                s = line.strip().split()\n",
    "                if len(s) == word_dim + 1:\n",
    "                    all_word_embeds[s[0]] = np.array([float(i) for i in s[1:]])\n",
    "\n",
    "            word_embeds = np.random.uniform(-np.sqrt(0.06), np.sqrt(0.06), (len(word_to_id), word_dim))\n",
    "\n",
    "            for w in word_to_id:\n",
    "                if w in all_word_embeds:\n",
    "                    word_embeds[word_to_id[w]] = all_word_embeds[w]\n",
    "                elif w.lower() in all_word_embeds:\n",
    "                    word_embeds[word_to_id[w]] = all_word_embeds[w.lower()]\n",
    "\n",
    "            print('Loaded %i pretrained embeddings.' % len(all_word_embeds))\n",
    "\n",
    "            with open(mapping_file, 'wb') as f:\n",
    "                mappings = {\n",
    "                    'word_to_id': word_to_id,\n",
    "                    'tag_to_id': tag_to_id,\n",
    "                    'id_to_tag': id_to_tag,\n",
    "                    'char_to_id': char_to_id,\n",
    "                    'parameters': parameters,\n",
    "                    'word_embeds': word_embeds\n",
    "                }\n",
    "                cPickle.dump(mappings, f)\n",
    "        else:\n",
    "            mappings = cPickle.load(open(mapping_file,'rb'))\n",
    "            \n",
    "        return train_data, dev_data, test_data, test_train_data, mappings\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: CNN_BiLSTM_CRF\n",
      "Dataset: conll\n",
      "Found 7518 unique words (203621 in total)\n",
      "Loading pretrained embeddings from wordvectors/glove.6B.100d.txt...\n",
      "Found 85 unique characters\n",
      "Found 11 unique named entity tags\n",
      "14041 / 3250 / 3453 sentences in train / dev / test.\n",
      "Loaded 400000 pretrained embeddings.\n",
      "Load Complete\n"
     ]
    }
   ],
   "source": [
    "parameters = OrderedDict()\n",
    "\n",
    "parameters['model'] = opt.usemodel\n",
    "parameters['wrdim'] = opt.worddim\n",
    "parameters['ptrnd'] = opt.pretrnd\n",
    "\n",
    "if opt.usemodel == 'CNN_BiLSTM_CRF':\n",
    "    parameters['lower'] = 1\n",
    "    parameters['zeros'] = 0\n",
    "    parameters['cpdim'] = 0\n",
    "    parameters['dpout'] = 0.5\n",
    "    parameters['chdim'] = 25\n",
    "    parameters['tgsch'] = 'iob'\n",
    "\n",
    "    parameters['wldim'] = 200\n",
    "    parameters['cldim'] = 25\n",
    "    parameters['cnchl'] = 25\n",
    "    \n",
    "    parameters['lrate'] = 0.015\n",
    "    \n",
    "elif opt.usemodel == 'CNN_BiLSTM_CRF_MC':\n",
    "    parameters['lower'] = 1\n",
    "    parameters['zeros'] = 0\n",
    "    parameters['cpdim'] = 0\n",
    "    parameters['dpout'] = 0.5\n",
    "    parameters['chdim'] = 25\n",
    "    parameters['tgsch'] = 'iobes'\n",
    "\n",
    "    parameters['wldim'] = 200\n",
    "    parameters['cldim'] = 25\n",
    "    parameters['cnchl'] = 25\n",
    "    \n",
    "    parameters['lrate'] = 0.015\n",
    "\n",
    "elif opt.usemodel == 'CNN_CNN_LSTM':\n",
    "    parameters['lower'] = 1\n",
    "    parameters['zeros'] = 0\n",
    "    parameters['cpdim'] = 0\n",
    "    parameters['dpout'] = 0.5\n",
    "    parameters['chdim'] = 25\n",
    "    parameters['tgsch'] = 'iobes'\n",
    "    \n",
    "    parameters['w1chl'] = 400\n",
    "    parameters['w2chl'] = 400\n",
    "    parameters['cldim'] = 25\n",
    "    parameters['cnchl'] = 50\n",
    "    parameters['dchid'] = 50\n",
    "    \n",
    "    parameters['lrate'] = 0.01\n",
    "    \n",
    "else:\n",
    "    raise NotImplementedError()\n",
    "\n",
    "use_dataset = opt.dataset\n",
    "dataset_path = os.path.join('datasets', use_dataset)\n",
    "result_path = os.path.join(opt.result_path, use_dataset)\n",
    "model_name = opt.usemodel\n",
    "model_load = opt.reload\n",
    "loader = Loader()\n",
    "\n",
    "print('Model:', model_name)\n",
    "print('Dataset:', use_dataset)\n",
    "\n",
    "if not os.path.exists(result_path):\n",
    "    os.makedirs(result_path)\n",
    "    \n",
    "if not os.path.exists(os.path.join(result_path,model_name)):\n",
    "    os.makedirs(os.path.join(result_path,model_name))\n",
    "\n",
    "if opt.dataset == 'conll':\n",
    "    train_data, dev_data, test_data, test_train_data, mappings = loader.load_conll(dataset_path, parameters)\n",
    "\n",
    "word_to_id = mappings['word_to_id']\n",
    "tag_to_id = mappings['tag_to_id']\n",
    "char_to_id = mappings['char_to_id']\n",
    "word_embeds = mappings['word_embeds']\n",
    "\n",
    "print('Load Complete')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'caps': [2, 2],\n",
       " 'chars': [[50, 1, 3, 1, 7], [44, 9, 2, 12, 29, 21, 13, 7, 4]],\n",
       " 'str_words': [u'Peter', u'Blackburn'],\n",
       " 'tags': [2, 4],\n",
       " 'words': [792, 1895]}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: u'O',\n",
       " 1: u'B-LOC',\n",
       " 2: u'B-PER',\n",
       " 3: u'B-ORG',\n",
       " 4: u'I-PER',\n",
       " 5: u'I-ORG',\n",
       " 6: u'B-MISC',\n",
       " 7: u'I-LOC',\n",
       " 8: u'I-MISC',\n",
       " 9: '<START>',\n",
       " 10: '<STOP>'}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mappings['id_to_tag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "class baseRNN(nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size, hidden_size, input_dropout_p, output_dropout_p, n_layers, rnn_cell, max_len=25):\n",
    "        super(baseRNN, self).__init__()\n",
    "        \n",
    "        self.vocab_size = vocab_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        self.max_len = max_len\n",
    "        \n",
    "        self.input_dropout_p = input_dropout_p\n",
    "        self.output_dropout_p = output_dropout_p\n",
    "        \n",
    "        if rnn_cell.lower() == 'lstm':\n",
    "            self.rnn_cell = nn.LSTM\n",
    "        elif rnn_cell.lower() == 'gru':\n",
    "            self.rnn_cell = nn.GRU\n",
    "        else:\n",
    "            raise ValueError(\"Unsupported RNN Cell: {0}\".format(rnn_cell))\n",
    "\n",
    "        self.input_dropout = nn.Dropout(p=input_dropout_p)\n",
    "\n",
    "    def forward(self, *args, **kwargs):\n",
    "        raise NotImplementedError()\n",
    "        \n",
    "class CharEncoderCNN(nn.Module):\n",
    "    \n",
    "    def __init__(self, vocab_size, embedding_size ,out_channels, kernel_width, pad_width, \n",
    "                 input_dropout_p=0, output_dropout_p=0, in_channels=1):\n",
    "        \n",
    "        super(CharEncoderCNN, self).__init__()\n",
    "        \n",
    "        self.out_channels = out_channels\n",
    "        self.input_dropout = nn.Dropout(input_dropout_p)\n",
    "        self.output_dropout = nn.Dropout(output_dropout_p)\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
    "        self.cnn = nn.Conv2d(in_channels, out_channels, kernel_size = (kernel_width, embedding_size),\n",
    "                             padding = (pad_width,0))\n",
    "\n",
    "    def forward(self, input_var, input_lengths=None):\n",
    "        \n",
    "        embedded = self.embedding(input_var).unsqueeze(1)\n",
    "        embedded = self.input_dropout(embedded)\n",
    "        output = self.cnn(embedded)\n",
    "        output = nn.functional.max_pool2d(output, kernel_size=(output.size(2), 1))\n",
    "        output = output.squeeze(3).squeeze(2)\n",
    "        \n",
    "        return output\n",
    "    \n",
    "class DecoderCRF(nn.Module):\n",
    "\n",
    "    def __init__(self, input_dimension, tag_to_ix, input_dropout_p=0.5):\n",
    "        \n",
    "        super(DecoderCRF, self).__init__()\n",
    "        \n",
    "        self.tag_to_ix = tag_to_ix\n",
    "        self.tagset_size = len(tag_to_ix)\n",
    "        \n",
    "        self.dropout = nn.Dropout(input_dropout_p)\n",
    "        self.hidden2tag = nn.Linear(input_dimension, self.tagset_size)\n",
    "        \n",
    "        self.transitions = nn.Parameter(torch.zeros(self.tagset_size, self.tagset_size))\n",
    "        self.transitions.data[tag_to_ix[START_TAG], :] = -10000\n",
    "        self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000\n",
    "    \n",
    "\n",
    "    def viterbi_decode(self, feats, mask ,usecuda = True, score_only= False):\n",
    "    \n",
    "        batch_size, sequence_len, num_tags = feats.size()\n",
    "        \n",
    "        assert num_tags == self.tagset_size\n",
    "        \n",
    "        mask = mask.transpose(0, 1).contiguous()\n",
    "        feats = feats.transpose(0, 1).contiguous()\n",
    "        \n",
    "        backpointers = []\n",
    "        \n",
    "        all_forward_vars = Variable(torch.Tensor(sequence_len, \n",
    "                                    batch_size, num_tags).fill_(0.)).cuda()\n",
    "        \n",
    "        init_vars = torch.Tensor(batch_size, num_tags).fill_(-10000.)\n",
    "        init_vars[:,self.tag_to_ix[START_TAG]] = 0.\n",
    "        if usecuda:\n",
    "            forward_var = Variable(init_vars).cuda()\n",
    "        else:\n",
    "            forward_var = Variable(init_vars)\n",
    "        \n",
    "        for i in range(sequence_len):\n",
    "            broadcast_forward = forward_var.view(batch_size, 1, num_tags)\n",
    "            transition_scores = self.transitions.view(1, num_tags, num_tags)\n",
    "            \n",
    "            next_tag_var = broadcast_forward + transition_scores\n",
    "            \n",
    "            viterbivars_t, bptrs_t = torch.max(next_tag_var, dim=2)\n",
    "            \n",
    "            forward_var = viterbivars_t + feats[i]\n",
    "            all_forward_vars[i,:,:] = forward_var\n",
    "\n",
    "            bptrs_t = bptrs_t.squeeze().data.cpu().numpy()\n",
    "            backpointers.append(bptrs_t)\n",
    "        \n",
    "        mask_sum = torch.sum(mask, dim = 0, keepdim =True) - 1\n",
    "        mask_sum_ex = mask_sum.view(1, batch_size, 1).expand(1, batch_size, num_tags)\n",
    "        final_forward_var = all_forward_vars.gather(0, mask_sum_ex).squeeze(0)\n",
    "        \n",
    "        terminal_var = final_forward_var + self.transitions[self.tag_to_ix[STOP_TAG]].view(1, num_tags)\n",
    "        terminal_var.data[:,self.tag_to_ix[STOP_TAG]] = -10000.\n",
    "        terminal_var.data[:,self.tag_to_ix[START_TAG]] = -10000.\n",
    "        \n",
    "        path_score, best_tag_id = torch.max(terminal_var, dim = 1)\n",
    "                \n",
    "        if score_only:\n",
    "            return path_score\n",
    "        \n",
    "        n_mask_sum = mask_sum.squeeze().data.cpu().numpy() + 1\n",
    "        best_tag_id = best_tag_id.data.cpu().numpy()\n",
    "        decoded_tags = []\n",
    "        for i in range(batch_size):\n",
    "            best_path = [best_tag_id[i]]\n",
    "            bp_list = reversed([itm[i] for itm in backpointers[:n_mask_sum[i]]])\n",
    "            for bptrs_t in bp_list:\n",
    "                best_tag_id[i] = bptrs_t[best_tag_id[i]]\n",
    "                best_path.append(best_tag_id[i])\n",
    "            start = best_path.pop()\n",
    "            assert start == self.tag_to_ix[START_TAG]\n",
    "            best_path.reverse()\n",
    "            decoded_tags.append(best_path)\n",
    "        \n",
    "        return path_score, decoded_tags\n",
    "    \n",
    "    def crf_forward(self, feats, mask, usecuda=True):\n",
    "        \n",
    "        batch_size, sequence_length, num_tags = feats.size()\n",
    "        \n",
    "        mask = mask.float().transpose(0, 1).contiguous()\n",
    "        feats = feats.transpose(0, 1).contiguous()\n",
    "        \n",
    "        init_alphas = torch.Tensor(batch_size, num_tags).fill_(-10000.)\n",
    "        init_alphas[:,self.tag_to_ix[START_TAG]] = 0.\n",
    "        if usecuda:\n",
    "            forward_var = Variable(init_alphas).cuda()\n",
    "        else:\n",
    "            forward_var = Variable(init_alphas)\n",
    "        \n",
    "        for i in range(sequence_length):\n",
    "            emit_score = feats[i].view(batch_size, num_tags, 1)\n",
    "            transition_scores = self.transitions.view(1, num_tags, num_tags)\n",
    "            broadcast_forward = forward_var.view(batch_size, 1, num_tags)\n",
    "            tag_var = broadcast_forward + transition_scores + emit_score \n",
    "            \n",
    "            forward_var = (log_sum_exp(tag_var, dim = 2) * mask[i].view(batch_size, 1) +\n",
    "                            forward_var * (1 - mask[i]).view(batch_size, 1))\n",
    "            \n",
    "        terminal_var = (forward_var + (self.transitions[self.tag_to_ix[STOP_TAG]]).view(1, -1))\n",
    "        alpha = log_sum_exp(terminal_var, dim = 1)\n",
    "        \n",
    "        return alpha\n",
    "        \n",
    "    \n",
    "    def score_sentence(self, feats, tags, mask, usecuda=True):\n",
    "                \n",
    "        batch_size, sequence_length, num_tags = feats.size()\n",
    "        \n",
    "        feats = feats.transpose(0, 1).contiguous()\n",
    "        tags = tags.transpose(0, 1).contiguous()\n",
    "        mask = mask.float().transpose(0, 1).contiguous()\n",
    "                \n",
    "        broadcast_transitions = self.transitions.view(1, num_tags, num_tags).expand(batch_size, num_tags, num_tags)\n",
    "        \n",
    "        score = self.transitions[:,self.tag_to_ix[START_TAG]].index_select(0, tags[0])\n",
    "        \n",
    "        for i in range(sequence_length - 1):\n",
    "            current_tag, next_tag = tags[i], tags[i+1]\n",
    "            \n",
    "            transition_score = (\n",
    "                     broadcast_transitions\n",
    "                    .gather(1, next_tag.view(batch_size, 1, 1).expand(batch_size, 1, num_tags))\n",
    "                    .squeeze(1)\n",
    "                    .gather(1, current_tag.view(batch_size, 1))\n",
    "                    .squeeze(1)\n",
    "                    )\n",
    "\n",
    "            emit_score = feats[i].gather(1, current_tag.view(batch_size, 1)).squeeze(1)\n",
    "\n",
    "            score = score + transition_score* mask[i + 1] + emit_score * mask[i]  \n",
    "        last_tag_index = mask.sum(0).long() - 1\n",
    "\n",
    "        last_tags = tags.gather(0, last_tag_index.view(1, batch_size).expand(sequence_length, batch_size))\n",
    "        last_tags = last_tags[0]\n",
    "\n",
    "        last_transition_score = self.transitions[self.tag_to_ix[STOP_TAG]].index_select(0, last_tags)\n",
    "        \n",
    "        last_inputs = feats[-1]                                     \n",
    "        last_input_score = last_inputs.gather(1, last_tags.view(batch_size, 1))\n",
    "        last_input_score = last_input_score.squeeze(1)\n",
    "        \n",
    "        score = score + last_transition_score + last_input_score * mask[-1]\n",
    "        \n",
    "        return score\n",
    "    \n",
    "    def decode(self, input_var, mask, usecuda=True):\n",
    "        \n",
    "        input_var = self.dropout(input_var)\n",
    "        features = self.hidden2tag(input_var)\n",
    "        score, tag_seq = self.viterbi_decode(features, mask, usecuda=usecuda)\n",
    "        \n",
    "        return score, tag_seq\n",
    "    \n",
    "    def forward(self, input_var, tags, mask=None, usecuda=True):\n",
    "        \n",
    "        if mask is None:\n",
    "            mask = torch.autograd.Variable(torch.ones(*tags.size()).long())\n",
    "        \n",
    "        input_var = self.dropout(input_var)\n",
    "        features = self.hidden2tag(input_var)\n",
    "        forward_score = self.crf_forward(features, mask, usecuda=usecuda)\n",
    "        ground_score = self.score_sentence(features, tags, mask, usecuda=usecuda)\n",
    "        \n",
    "        return forward_score-ground_score\n",
    "\n",
    "\n",
    "class WordEncoderRNN(baseRNN):\n",
    "\n",
    "    def __init__(self, vocab_size, embedding_size ,hidden_size, char_size, cap_size, input_dropout_p=0.5, \n",
    "                 output_dropout_p=0, n_layers=1, bidirectional=True, rnn_cell='lstm'):\n",
    "        \n",
    "        super(WordEncoderRNN, self).__init__(vocab_size, hidden_size, input_dropout_p, \n",
    "                                             output_dropout_p, n_layers, rnn_cell)\n",
    "\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
    "        \n",
    "        augmented_embedding_size = embedding_size + char_size + cap_size\n",
    "        self.rnn = self.rnn_cell(augmented_embedding_size, hidden_size, n_layers,\n",
    "                                 bidirectional=bidirectional, dropout=output_dropout_p,\n",
    "                                 batch_first=True)\n",
    "\n",
    "    def forward(self, words, char_embedding, cap_embedding, input_lengths):\n",
    "        \n",
    "        embedded = self.embedding(words)\n",
    "        if cap_embedding is not None:\n",
    "            embedded = torch.cat((embedded,char_embedding,cap_embedding),2)  \n",
    "        else:\n",
    "            embedded = torch.cat((embedded,char_embedding),2)\n",
    "    \n",
    "        embedded = self.input_dropout(embedded)\n",
    "        embedded = nn.utils.rnn.pack_padded_sequence(embedded, input_lengths, batch_first= True)\n",
    "        output, _ = self.rnn(embedded)\n",
    "        output, _ = nn.utils.rnn.pad_packed_sequence(output, batch_first= True)\n",
    "        \n",
    "        return output\n",
    "\n",
    "class WordEncoderCNN(nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size, embedding_size, char_size, kernel_width = 5, pad_width = 4, \n",
    "                 in_channels=1, out1_channels=800, out2_channels=800, cap_size=0, input_dropout_p=0.5, \n",
    "                 output_dropout_p=0):\n",
    "        \n",
    "        super(WordEncoderCNN, self).__init__()\n",
    "        \n",
    "        self.kernel_width = kernel_width\n",
    "        self.out2_channels = out2_channels\n",
    "        self.input_dropout = nn.Dropout(p=input_dropout_p)\n",
    "        \n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
    "        new_embedding_size = embedding_size + char_size\n",
    "        self.cnn1 = nn.Conv2d(in_channels, out1_channels, kernel_size=(kernel_width, new_embedding_size),\n",
    "                             padding = (pad_width,0))\n",
    "        self.cnn2 = nn.Conv2d(out1_channels, out2_channels, kernel_size=(kernel_width, 1),\n",
    "                             padding = (pad_width,0))\n",
    "\n",
    "    def forward(self, words, char_embedding, cap_embedding=None ,input_lengths=None):\n",
    "        \n",
    "        embedded = self.embedding(words)\n",
    "        \n",
    "        if cap_embedding:\n",
    "            embedded = torch.cat((embedded,char_embedding,cap_embedding),2)  \n",
    "        else:\n",
    "            embedded = torch.cat((embedded,char_embedding),2)\n",
    "        \n",
    "        embedded1 = embedded.unsqueeze(1)\n",
    "        embedded1 = self.input_dropout(embedded1)\n",
    "                        \n",
    "        output1 = self.cnn1(embedded1)\n",
    "        output1 = nn.functional.max_pool2d(output1, kernel_size=(self.kernel_width, 1), stride = 1)\n",
    "        \n",
    "        output2 = self.cnn2(output1)\n",
    "        output2 = nn.functional.max_pool2d(output2, kernel_size=(self.kernel_width, 1), stride = 1)\n",
    "        output2 = output2.squeeze(3).transpose(1,2)\n",
    "        \n",
    "        return output2, embedded\n",
    "\n",
    "\n",
    "class DecoderRNN(nn.Module):\n",
    "    def __init__(self, input_size ,hidden_size, tag_size, tag_to_ix, input_dropout_p=0.5, \n",
    "                 output_dropout_p=0, n_layers=1):\n",
    "        super(DecoderRNN, self).__init__()\n",
    "        \n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        \n",
    "        self.input_dropout_p = input_dropout_p\n",
    "        self.output_dropout_p = output_dropout_p\n",
    "        \n",
    "        self.tag_to_ix = tag_to_ix\n",
    "        self.tagset_size = len(tag_to_ix)\n",
    "        \n",
    "        self.dropout = nn.Dropout(input_dropout_p)\n",
    "        \n",
    "        self.rnn = nn.LSTM(input_size + tag_size, hidden_size, n_layers, bidirectional=False)\n",
    "        self.linear = nn.Linear(hidden_size, tag_size)\n",
    "        self.ignore = -1\n",
    "        self.lossfunc = nn.CrossEntropyLoss(ignore_index= self.ignore)\n",
    "        \n",
    "    def forward_step(self, input_var, prev_tag, hidden ,usecuda=True):\n",
    "        \n",
    "        prev_tag_onehot = torch.eye(self.tagset_size)\n",
    "        prev_tag_onehot = prev_tag_onehot.index_select(0,torch.LongTensor(prev_tag))\n",
    "        \n",
    "        if usecuda:\n",
    "            prev_tag_onehot = Variable(prev_tag_onehot).cuda()\n",
    "        else:\n",
    "            prev_tag_onehot = Variable(prev_tag_onehot)\n",
    "        \n",
    "        decoder_input = torch.cat([input_var, prev_tag_onehot],1).unsqueeze(0)\n",
    "        output, hidden = self.rnn(decoder_input, hidden)\n",
    "        output = self.linear(output.squeeze(0))\n",
    "        output_tag = output.max(1)[1].data.cpu().numpy().tolist()\n",
    "\n",
    "        return output, output_tag, hidden\n",
    "        \n",
    "    def forward(self, input_var, tags, mask, usecuda=True):\n",
    "        \n",
    "        batch_size, sequence_len, _ = input_var.size()\n",
    "        \n",
    "        input_var = self.dropout(input_var)\n",
    "        \n",
    "        input_var = input_var.transpose(0, 1).contiguous()\n",
    "        \n",
    "        tags = tags.transpose(0, 1).contiguous()\n",
    "        mask = mask.float().transpose(0, 1).contiguous()\n",
    "        \n",
    "        maskedtags = tags.clone()\n",
    "        maskedtags[mask==0] = -1\n",
    "        \n",
    "        loss = 0.0\n",
    "        prev_tag = [self.tag_to_ix[START_TAG]]*batch_size\n",
    "        hidden = None\n",
    "        \n",
    "        for i in range(sequence_len):\n",
    "            output, prev_tag, hidden=self.forward_step(input_var[i], prev_tag, hidden, \n",
    "                                                       usecuda=usecuda)\n",
    "            loss += self.lossfunc(output, maskedtags[i])\n",
    "        return loss\n",
    "    \n",
    "    def decode(self, input_var, wordslen, usecuda=True):\n",
    "        \n",
    "        batch_size, sequence_len, _ = input_var.size()\n",
    "        \n",
    "        input_var = self.dropout(input_var)\n",
    "        input_var = input_var.transpose(0, 1).contiguous()\n",
    "        \n",
    "        loss = 0.0\n",
    "        prev_tag = [self.tag_to_ix[START_TAG]]*batch_size\n",
    "        hidden = None\n",
    "        \n",
    "        tag_seq = []\n",
    "        probs= []\n",
    "        for i in range(sequence_len):\n",
    "            output, prev_tag, hidden=self.forward_step(input_var[i], prev_tag, hidden, \n",
    "                                                       usecuda=usecuda)\n",
    "            tag_seq.append(prev_tag)\n",
    "            pb = nn.functional.softmax(output, dim = 1).data.cpu().numpy()\n",
    "            probs.append(pb)\n",
    "        \n",
    "        probs = np.array(probs).transpose(1,0,2)\n",
    "        \n",
    "        tag_seq = np.array(tag_seq).transpose().tolist()\n",
    "        tag_seq = [ts[:wordslen[i]] for i,ts in enumerate(tag_seq)]\n",
    "        \n",
    "        return probs, tag_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN_CNN_LSTM(nn.Module):\n",
    "    \n",
    "    def __init__(self, word_vocab_size, word_embedding_dim, word_out1_channels, word_out2_channels,\n",
    "                 char_vocab_size, char_embedding_dim, char_out_channels, decoder_hidden_units,\n",
    "                 tag_to_id, cap_input_dim=4, cap_embedding_dim=0, pretrained=None):\n",
    "        \n",
    "        super(CNN_CNN_LSTM, self).__init__()\n",
    "        \n",
    "        self.word_vocab_size = word_vocab_size\n",
    "        self.word_embedding_dim = word_embedding_dim\n",
    "        self.word_out1_channels = word_out1_channels\n",
    "        self.word_out2_channels = word_out2_channels\n",
    "        \n",
    "        self.char_vocab_size = char_vocab_size\n",
    "        self.char_embedding_dim = char_embedding_dim\n",
    "        self.char_out_channels = char_out_channels\n",
    "        \n",
    "        self.cap_input_dim = cap_input_dim\n",
    "        self.cap_embedding_dim = cap_embedding_dim\n",
    "        \n",
    "        self.tag_to_ix = tag_to_id\n",
    "        self.tagset_size = len(tag_to_id)\n",
    "        \n",
    "        self.initializer = Initializer()\n",
    "        self.loader = Loader()\n",
    "        \n",
    "        if self.cap_input_dim and self.cap_embedding_dim:\n",
    "            self.cap_embedder = nn.Embedding(self.cap_input_dim, self.cap_embedding_dim)\n",
    "            self.initializer.init_embedding(self.cap_embedder.weight)\n",
    "        \n",
    "        self.char_encoder = CharEncoderCNN(char_vocab_size, char_embedding_dim, char_out_channels, \n",
    "                                           kernel_width=3, pad_width=1)\n",
    "        \n",
    "        self.initializer.init_embedding(self.char_encoder.embedding.weight)\n",
    "        \n",
    "        self.word_encoder = WordEncoderCNN(word_vocab_size, word_embedding_dim, char_out_channels,\n",
    "                                           kernel_width = 3, pad_width = 2, input_dropout_p=0.5,\n",
    "                                           out1_channels=word_out1_channels, out2_channels=word_out2_channels)\n",
    "        \n",
    "        if pretrained is not None:\n",
    "            self.word_encoder.embedding.weight = nn.Parameter(torch.FloatTensor(pretrained))\n",
    "        \n",
    "        augmented_decoder_inp_size = (word_out2_channels + word_embedding_dim + \n",
    "                                      char_out_channels + cap_embedding_dim)\n",
    "        self.decoder = DecoderRNN(augmented_decoder_inp_size, decoder_hidden_units, self.tagset_size, \n",
    "                                  self.tag_to_ix, input_dropout_p=0.5)\n",
    "        \n",
    "    def forward(self, words, tags, chars, caps, wordslen, charslen, tagsmask, usecuda=True):\n",
    "        \n",
    "        batch_size, max_len = words.size()\n",
    "        \n",
    "        cap_features = self.cap_embedder(caps) if self.cap_embedding_dim else None\n",
    "        \n",
    "        char_features = self.char_encoder(chars)\n",
    "        char_features = char_features.view(batch_size, max_len, -1)\n",
    "        \n",
    "        word_features, word_input_feats = self.word_encoder(words, char_features, cap_features)\n",
    "        \n",
    "        new_word_features = torch.cat((word_features,word_input_feats),2)\n",
    "        loss = self.decoder(new_word_features, tags, tagsmask, usecuda=usecuda)\n",
    "        \n",
    "        return loss\n",
    "    \n",
    "    def decode(self, words, chars, caps, wordslen, charslen, tagsmask, usecuda=True, \n",
    "               score_only = False):\n",
    "        \n",
    "        batch_size, max_len = words.size()\n",
    "        \n",
    "        cap_features = self.cap_embedder(caps) if self.cap_embedding_dim else None\n",
    "        \n",
    "        char_features = self.char_encoder(chars)\n",
    "        char_features = char_features.view(batch_size, max_len, -1)\n",
    "        \n",
    "        word_features, word_input_feats = self.word_encoder(words, char_features, cap_features)\n",
    "        \n",
    "        new_word_features = torch.cat((word_features,word_input_feats),2)\n",
    "        \n",
    "        if score_only:\n",
    "            score, _ = self.decoder.decode(new_word_features, wordslen, usecuda=usecuda)\n",
    "            return score\n",
    "        \n",
    "        score, tag_seq = self.decoder.decode(new_word_features, wordslen, usecuda=usecuda)\n",
    "        return score, tag_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building Model............................................................................\n",
      "CNN_CNN_LSTM\n",
      "Initial learning rate is: 0.01\n"
     ]
    }
   ],
   "source": [
    "if model_load:\n",
    "    print ('Loading Saved Weights....................................................................')\n",
    "    model_path = os.path.join(result_path, model_name, opt.checkpoint, 'modelweights')\n",
    "    model = torch.load(model_path)\n",
    "else:\n",
    "    print('Building Model............................................................................')\n",
    "    if (model_name == 'CNN_BiLSTM_CRF'):\n",
    "        print ('CNN_BiLSTM_CRF')\n",
    "        word_vocab_size = len(word_to_id)\n",
    "        word_embedding_dim = parameters['wrdim']\n",
    "        word_hidden_dim = parameters['wldim']\n",
    "        char_vocab_size = len(char_to_id)\n",
    "        char_embedding_dim = parameters['chdim']\n",
    "        char_out_channels = parameters['cnchl']\n",
    "\n",
    "        model = CNN_BiLSTM_CRF(word_vocab_size, word_embedding_dim, word_hidden_dim, char_vocab_size,\n",
    "                               char_embedding_dim, char_out_channels, tag_to_id, pretrained = word_embeds,\n",
    "                               cap_embedding_dim = 10)\n",
    "        \n",
    "    elif (model_name == 'CNN_BiLSTM_CRF_MC'):\n",
    "        print ('CNN_BiLSTM_CRF_MC')\n",
    "        word_vocab_size = len(word_to_id)\n",
    "        word_embedding_dim = parameters['wrdim']\n",
    "        word_hidden_dim = parameters['wldim']\n",
    "        char_vocab_size = len(char_to_id)\n",
    "        char_embedding_dim = parameters['chdim']\n",
    "        char_out_channels = parameters['cnchl']\n",
    "\n",
    "        model = CNN_BiLSTM_CRF_MC(word_vocab_size, word_embedding_dim, word_hidden_dim, char_vocab_size,\n",
    "                               char_embedding_dim, char_out_channels, tag_to_id, pretrained = word_embeds)\n",
    "\n",
    "    elif (model_name == 'CNN_CNN_LSTM'):\n",
    "        print ('CNN_CNN_LSTM')\n",
    "        word_vocab_size = len(word_to_id)\n",
    "        word_embedding_dim = parameters['wrdim']\n",
    "        word_out1_channels = parameters['w1chl']\n",
    "        word_out2_channels = parameters['w2chl']\n",
    "        char_vocab_size = len(char_to_id)\n",
    "        char_embedding_dim = parameters['chdim']\n",
    "        char_out_channels = parameters['cnchl']\n",
    "        decoder_hidden_units = parameters['dchid']\n",
    "\n",
    "        model = CNN_CNN_LSTM(word_vocab_size, word_embedding_dim, word_out1_channels, word_out2_channels,\n",
    "                             char_vocab_size, char_embedding_dim, char_out_channels, decoder_hidden_units,\n",
    "                             tag_to_id, pretrained = word_embeds)\n",
    "    \n",
    "    \n",
    "model.cuda()\n",
    "learning_rate = parameters['lrate']\n",
    "print('Initial learning rate is: %s' %(learning_rate))\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import codecs\n",
    "\n",
    "class Evaluator(object):\n",
    "    def __init__(self, result_path, model_name, mappings, usecuda=True):\n",
    "        self.result_path = result_path\n",
    "        self.model_name = model_name\n",
    "        self.tag_to_id = mappings['tag_to_id']\n",
    "        self.id_to_tag = mappings['id_to_tag']\n",
    "        self.usecuda = usecuda\n",
    "\n",
    "    def evaluate_conll(self, model, dataset, best_F, eval_script='./datasets/conll/conlleval',\n",
    "                       checkpoint_folder='.', record_confmat = False, batch_size = 32):\n",
    "        \n",
    "        prediction = []\n",
    "        save = False\n",
    "        new_F = 0.0\n",
    "        confusion_matrix = torch.zeros((len(self.tag_to_id) - 2, len(self.tag_to_id) - 2))\n",
    "    \n",
    "        data_batches = create_batches(dataset, batch_size = batch_size, str_words = True,\n",
    "                                      tag_padded = False)\n",
    "\n",
    "        for data in data_batches:\n",
    "\n",
    "            words = data['words']\n",
    "            chars = data['chars']\n",
    "            caps = data['caps']\n",
    "            mask = data['tagsmask']\n",
    "\n",
    "            if self.usecuda:\n",
    "                words = Variable(torch.LongTensor(words)).cuda()\n",
    "                chars = Variable(torch.LongTensor(chars)).cuda()\n",
    "                caps = Variable(torch.LongTensor(caps)).cuda()\n",
    "                mask = Variable(torch.LongTensor(mask)).cuda()\n",
    "            else:\n",
    "                words = Variable(torch.LongTensor(words))\n",
    "                chars = Variable(torch.LongTensor(chars))\n",
    "                caps = Variable(torch.LongTensor(caps))\n",
    "                mask = Variable(torch.LongTensor(mask))\n",
    "\n",
    "            wordslen = data['wordslen']\n",
    "            charslen = data['charslen']\n",
    "            \n",
    "            str_words = data['str_words']\n",
    "            \n",
    "            _, out = model.decode(words, chars, caps, wordslen, charslen, mask, usecuda = self.usecuda)\n",
    "            print (out)\n",
    "            assert False\n",
    "            \n",
    "            ground_truth_id = data['tags']\n",
    "            predicted_id = out            \n",
    "            \n",
    "            for (swords, sground_truth_id, spredicted_id) in zip(str_words, ground_truth_id, predicted_id):\n",
    "                for (word, true_id, pred_id) in zip(swords, sground_truth_id, spredicted_id):\n",
    "                    line = ' '.join([word, self.id_to_tag[true_id], self.id_to_tag[pred_id]])\n",
    "                    prediction.append(line)\n",
    "                    confusion_matrix[true_id, pred_id] += 1\n",
    "                prediction.append('')\n",
    "\n",
    "        predf = os.path.join(self.result_path, self.model_name, checkpoint_folder ,'pred.txt')\n",
    "        scoref = os.path.join(self.result_path, self.model_name, checkpoint_folder ,'score.txt')\n",
    "\n",
    "        with open(predf, 'wb') as f:\n",
    "            f.write('\\n'.join(prediction))\n",
    "\n",
    "        os.system('%s < %s > %s' % (eval_script, predf, scoref))\n",
    "\n",
    "        eval_lines = [l.rstrip() for l in codecs.open(scoref, 'r', 'utf8')]\n",
    "\n",
    "        for i, line in enumerate(eval_lines):\n",
    "            print(line)\n",
    "            if i == 1:\n",
    "                new_F = float(line.strip().split()[-1])\n",
    "                if new_F > best_F:\n",
    "                    best_F = new_F\n",
    "                    save = True\n",
    "                    print('the best F is ', new_F)\n",
    "        if record_confmat:\n",
    "            print((\"{: >2}{: >7}{: >7}%s{: >9}\" % (\"{: >7}\" * confusion_matrix.size(0))).format(\n",
    "                \"ID\", \"NE\", \"Total\",\n",
    "                *([self.id_to_tag[i] for i in range(confusion_matrix.size(0))] + [\"Percent\"])\n",
    "            ))\n",
    "            for i in range(confusion_matrix.size(0)):\n",
    "                print((\"{: >2}{: >7}{: >7}%s{: >9}\" % (\"{: >7}\" * confusion_matrix.size(0))).format(\n",
    "                    str(i), self.id_to_tag[i], str(confusion_matrix[i].sum()),\n",
    "                    *([confusion_matrix[i][j] for j in range(confusion_matrix.size(0))] +\n",
    "                      [\"%.3f\" % (confusion_matrix[i][i] * 100. / max(1, confusion_matrix[i].sum()))])\n",
    "                ))\n",
    "            \n",
    "        return best_F, new_F, save\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import time\n",
    "import sys\n",
    "import os\n",
    "\n",
    "class Trainer(object):\n",
    "    \n",
    "    def __init__(self, model, optimizer, result_path, model_name, usedataset, mappings, \n",
    "                 eval_every=1, usecuda = True):\n",
    "        self.model = model\n",
    "        self.optimizer = optimizer\n",
    "        self.eval_every = eval_every\n",
    "        self.model_name = os.path.join(result_path, model_name)\n",
    "        self.usecuda = usecuda\n",
    "        \n",
    "        if usedataset=='conll':\n",
    "            self.evaluator = Evaluator(result_path, model_name, mappings, usecuda).evaluate_conll\n",
    "    \n",
    "    def adjust_learning_rate(self, optimizer, lr):\n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = lr\n",
    "        \n",
    "            \n",
    "    def train_model(self, num_epochs, train_data, dev_data, test_train_data, test_data, learning_rate,\n",
    "                    checkpoint_folder='.', eval_test_train=True, plot_every=1, adjust_lr=True,\n",
    "                    batch_size = 2):\n",
    "        \n",
    "        losses = []\n",
    "        loss = 0.0\n",
    "        best_dev_F = -1.0\n",
    "        best_test_F = -1.0\n",
    "        best_train_F = -1.0\n",
    "        all_F=[[0,0,0]]\n",
    "        count = 0\n",
    "        word_count = 0\n",
    "        \n",
    "        self.model.train(True)\n",
    "        for epoch in range(1, num_epochs+1):\n",
    "            t=time.time()\n",
    "            \n",
    "            #Random Batching Ensure\n",
    "            train_batches = create_batches(train_data, batch_size= batch_size)\n",
    "            #Random Permutation instead of Range.\n",
    "            for i, index in enumerate(np.arange(len(train_batches[:5]))):\n",
    "                \n",
    "                data = train_batches[index]\n",
    "                self.model.zero_grad()\n",
    "\n",
    "                words = data['words']\n",
    "                tags = data['tags']\n",
    "                chars = data['chars']\n",
    "                caps = data['caps']\n",
    "                mask = data['tagsmask']\n",
    "                \n",
    "                if self.usecuda:\n",
    "                    words = Variable(torch.LongTensor(words)).cuda()\n",
    "                    chars = Variable(torch.LongTensor(chars)).cuda()\n",
    "                    caps = Variable(torch.LongTensor(caps)).cuda()\n",
    "                    mask = Variable(torch.LongTensor(mask)).cuda()\n",
    "                    tags = Variable(torch.LongTensor(tags)).cuda()\n",
    "                else:\n",
    "                    words = Variable(torch.LongTensor(words))\n",
    "                    chars = Variable(torch.LongTensor(chars))\n",
    "                    caps = Variable(torch.LongTensor(caps))\n",
    "                    mask = Variable(torch.LongTensor(mask))\n",
    "                    tags = Variable(torch.LongTensor(tags))\n",
    "                \n",
    "                wordslen = data['wordslen']\n",
    "                charslen = data['charslen']\n",
    "                batch_score = self.model(words, tags, chars, caps, wordslen, charslen, mask,\n",
    "                                         usecuda=self.usecuda)\n",
    "                loss += np.mean(batch_score.data.cpu().numpy()/np.array(data['wordslen']))\n",
    "                score = torch.sum(batch_score)\n",
    "                score.backward()\n",
    "                \n",
    "                \n",
    "                nn.utils.clip_grad_norm(self.model.parameters(), 5.0)\n",
    "                self.optimizer.step()\n",
    "                \n",
    "                count += 1\n",
    "                word_count += len(data['words'])\n",
    "                \n",
    "                if count % plot_every == 0:\n",
    "                    loss /= plot_every\n",
    "                    print(word_count, ': ', loss)\n",
    "                    if losses == []:\n",
    "                        losses.append(loss)\n",
    "                    losses.append(loss)\n",
    "                    loss = 0.0\n",
    "                    \n",
    "            if adjust_lr:\n",
    "                self.adjust_learning_rate(self.optimizer, lr=learning_rate/(1+0.05*count/len(train_data)))\n",
    "            \n",
    "            if epoch%self.eval_every==0:\n",
    "                \n",
    "                self.model.train(False)\n",
    "                \n",
    "                if eval_test_train:\n",
    "                    best_train_F, new_train_F, _ = self.evaluator(self.model, test_train_data, best_train_F, \n",
    "                                                                  checkpoint_folder=checkpoint_folder)\n",
    "                else:\n",
    "                    best_train_F, new_train_F, _ = 0, 0, 0\n",
    "                best_dev_F, new_dev_F, save = self.evaluator(self.model, dev_data, best_dev_F,\n",
    "                                                             checkpoint_folder=checkpoint_folder)\n",
    "                if save:\n",
    "                    torch.save(self.model, os.path.join(self.model_name, checkpoint_folder, 'modelweights'))\n",
    "                best_test_F, new_test_F, _ = self.evaluator(self.model, test_data, best_test_F,\n",
    "                                                            checkpoint_folder=checkpoint_folder)\n",
    "                sys.stdout.flush()\n",
    "\n",
    "                all_F.append([new_train_F, new_dev_F, new_test_F])\n",
    "                \n",
    "                self.model.train(True)\n",
    "\n",
    "            print('*'*80)\n",
    "            print('Epoch %d Complete: Time Taken %d' %(epoch ,time.time() - t))\n",
    "\n",
    "        return losses, all_F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "2 :  8.404297722710503\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "4 :  23.722637939453126\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "6 :  2.8609290994623655\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "8 :  3.2037055015563967\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "torch.Size([2, 19])\n",
      "10 :  2.536739235218888\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "torch.Size([32, 19])\n",
      "(32, 47, 19)\n",
      "[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0]]\n"
     ]
    },
    {
     "ename": "AssertionError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-208-cace9489dfd6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0musedataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmappings\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mmappings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m losses, all_F = trainer.train_model(opt.num_epochs, train_data, dev_data, test_train_data, test_data,\n\u001b[0;32m----> 3\u001b[0;31m                                      learning_rate = learning_rate)\n\u001b[0m",
      "\u001b[0;32m<ipython-input-207-3cf8121f91e8>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, num_epochs, train_data, dev_data, test_train_data, test_data, learning_rate, checkpoint_folder, eval_test_train, plot_every, adjust_lr, batch_size)\u001b[0m\n\u001b[1;32m     98\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0meval_test_train\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                     best_train_F, new_train_F, _ = self.evaluator(self.model, test_train_data, best_train_F, \n\u001b[0;32m--> 100\u001b[0;31m                                                                   checkpoint_folder=checkpoint_folder)\n\u001b[0m\u001b[1;32m    101\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m                     \u001b[0mbest_train_F\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_train_F\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-206-730cba96aadc>\u001b[0m in \u001b[0;36mevaluate_conll\u001b[0;34m(self, model, dataset, best_F, eval_script, checkpoint_folder, record_confmat, batch_size)\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwordslen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcharslen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0musecuda\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecuda\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0;32mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m             \u001b[0;32massert\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0mground_truth_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'tags'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "trainer = Trainer(model, optimizer, result_path, model_name, usedataset=opt.dataset, mappings= mappings) \n",
    "losses, all_F = trainer.train_model(opt.num_epochs, train_data, dev_data, test_train_data, test_data,\n",
    "                                     learning_rate = learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zero_digits(s):\n",
    "    \"\"\"\n",
    "    Replace every digit in a string by a zero.\n",
    "    \"\"\"\n",
    "    return re.sub('\\d', '0', s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import codecs\n",
    "from neural_ner.util.loader import Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = Loader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {}\n",
    "parameters['zeros'] = 0\n",
    "parameters['lower'] = 1\n",
    "parameters['wrdim'] = 100\n",
    "parameters['ptrnd'] = 'wordvectors/glove.6B.100d.txt'\n",
    "parameters['tgsch'] = 'iobes'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 20676 unique words (1633660 in total)\n",
      "Loading pretrained embeddings from wordvectors/glove.6B.100d.txt...\n",
      "Found 115 unique characters\n",
      "Found 75 unique named entity tags\n",
      "82122 / 12678 / 8968 sentences in train / dev / test.\n"
     ]
    }
   ],
   "source": [
    "train_data, dev_data, test_data, mappings = loader.load_ontonotes('datasets/ontonotes/',parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = []\n",
    "for data in train_data:\n",
    "    for word in data['str_words']:\n",
    "        prediction.append(word)\n",
    "with open('hawa.txt', 'wb') as f:\n",
    "    f.write('\\n'.join(prediction).encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "a= torch.randn(3,4,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 5, 4])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.transpose(1,2).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4.23408643, 3.74352882, 4.63904877])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.max(2).sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.autograd.Variable(torch.randn(3,5)).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.6069078 , -2.66035119, -0.99708431,  3.21097201,  1.12540611],\n",
       "       [ 0.11840909, -1.08448844, -0.24090727, -1.20281994, -0.29294011],\n",
       "       [-0.2129307 ,  0.29792313, -0.45437451,  0.00458517, -0.5133353 ]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.cpu().data.numpy() * np.random.randn(3,5,4).max(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch_p27]",
   "language": "python",
   "name": "conda-env-pytorch_p27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}