Chapter14 - Intro to LSTMs - Learn to Write Like Shakespeare.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class Tensor (object):\n",
    "    \n",
    "    def __init__(self,data,\n",
    "                 autograd=False,\n",
    "                 creators=None,\n",
    "                 creation_op=None,\n",
    "                 id=None):\n",
    "        \n",
    "        self.data = np.array(data)\n",
    "        self.autograd = autograd\n",
    "        self.grad = None\n",
    "\n",
    "        if(id is None):\n",
    "            self.id = np.random.randint(0,1000000000)\n",
    "        else:\n",
    "            self.id = id\n",
    "        \n",
    "        self.creators = creators\n",
    "        self.creation_op = creation_op\n",
    "        self.children = {}\n",
    "        \n",
    "        if(creators is not None):\n",
    "            for c in creators:\n",
    "                if(self.id not in c.children):\n",
    "                    c.children[self.id] = 1\n",
    "                else:\n",
    "                    c.children[self.id] += 1\n",
    "\n",
    "    def all_children_grads_accounted_for(self):\n",
    "        for id,cnt in self.children.items():\n",
    "            if(cnt != 0):\n",
    "                return False\n",
    "        return True \n",
    "        \n",
    "    def backward(self,grad=None, grad_origin=None):\n",
    "        if(self.autograd):\n",
    " \n",
    "            if(grad is None):\n",
    "                grad = Tensor(np.ones_like(self.data))\n",
    "\n",
    "            if(grad_origin is not None):\n",
    "                if(self.children[grad_origin.id] == 0):\n",
    "                    return\n",
    "                    print(self.id)\n",
    "                    print(self.creation_op)\n",
    "                    print(len(self.creators))\n",
    "                    for c in self.creators:\n",
    "                        print(c.creation_op)\n",
    "                    raise Exception(\"cannot backprop more than once\")\n",
    "                else:\n",
    "                    self.children[grad_origin.id] -= 1\n",
    "\n",
    "            if(self.grad is None):\n",
    "                self.grad = grad\n",
    "            else:\n",
    "                self.grad += grad\n",
    "            \n",
    "            # grads must not have grads of their own\n",
    "            assert grad.autograd == False\n",
    "            \n",
    "            # only continue backpropping if there's something to\n",
    "            # backprop into and if all gradients (from children)\n",
    "            # are accounted for override waiting for children if\n",
    "            # \"backprop\" was called on this variable directly\n",
    "            if(self.creators is not None and \n",
    "               (self.all_children_grads_accounted_for() or \n",
    "                grad_origin is None)):\n",
    "\n",
    "                if(self.creation_op == \"add\"):\n",
    "                    self.creators[0].backward(self.grad, self)\n",
    "                    self.creators[1].backward(self.grad, self)\n",
    "                    \n",
    "                if(self.creation_op == \"sub\"):\n",
    "                    self.creators[0].backward(Tensor(self.grad.data), self)\n",
    "                    self.creators[1].backward(Tensor(self.grad.__neg__().data), self)\n",
    "\n",
    "                if(self.creation_op == \"mul\"):\n",
    "                    new = self.grad * self.creators[1]\n",
    "                    self.creators[0].backward(new , self)\n",
    "                    new = self.grad * self.creators[0]\n",
    "                    self.creators[1].backward(new, self)                    \n",
    "                    \n",
    "                if(self.creation_op == \"mm\"):\n",
    "                    c0 = self.creators[0]\n",
    "                    c1 = self.creators[1]\n",
    "                    new = self.grad.mm(c1.transpose())\n",
    "                    c0.backward(new)\n",
    "                    new = self.grad.transpose().mm(c0).transpose()\n",
    "                    c1.backward(new)\n",
    "                    \n",
    "                if(self.creation_op == \"transpose\"):\n",
    "                    self.creators[0].backward(self.grad.transpose())\n",
    "\n",
    "                if(\"sum\" in self.creation_op):\n",
    "                    dim = int(self.creation_op.split(\"_\")[1])\n",
    "                    self.creators[0].backward(self.grad.expand(dim,\n",
    "                                                               self.creators[0].data.shape[dim]))\n",
    "\n",
    "                if(\"expand\" in self.creation_op):\n",
    "                    dim = int(self.creation_op.split(\"_\")[1])\n",
    "                    self.creators[0].backward(self.grad.sum(dim))\n",
    "                    \n",
    "                if(self.creation_op == \"neg\"):\n",
    "                    self.creators[0].backward(self.grad.__neg__())\n",
    "                    \n",
    "                if(self.creation_op == \"sigmoid\"):\n",
    "                    ones = Tensor(np.ones_like(self.grad.data))\n",
    "                    self.creators[0].backward(self.grad * (self * (ones - self)))\n",
    "                \n",
    "                if(self.creation_op == \"tanh\"):\n",
    "                    ones = Tensor(np.ones_like(self.grad.data))\n",
    "                    self.creators[0].backward(self.grad * (ones - (self * self)))\n",
    "                \n",
    "                if(self.creation_op == \"index_select\"):\n",
    "                    new_grad = np.zeros_like(self.creators[0].data)\n",
    "                    indices_ = self.index_select_indices.data.flatten()\n",
    "                    grad_ = grad.data.reshape(len(indices_), -1)\n",
    "                    for i in range(len(indices_)):\n",
    "                        new_grad[indices_[i]] += grad_[i]\n",
    "                    self.creators[0].backward(Tensor(new_grad))\n",
    "                    \n",
    "                if(self.creation_op == \"cross_entropy\"):\n",
    "                    dx = self.softmax_output - self.target_dist\n",
    "                    self.creators[0].backward(Tensor(dx))\n",
    "                    \n",
    "    def __add__(self, other):\n",
    "        if(self.autograd and other.autograd):\n",
    "            return Tensor(self.data + other.data,\n",
    "                          autograd=True,\n",
    "                          creators=[self,other],\n",
    "                          creation_op=\"add\")\n",
    "        return Tensor(self.data + other.data)\n",
    "\n",
    "    def __neg__(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data * -1,\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"neg\")\n",
    "        return Tensor(self.data * -1)\n",
    "    \n",
    "    def __sub__(self, other):\n",
    "        if(self.autograd and other.autograd):\n",
    "            return Tensor(self.data - other.data,\n",
    "                          autograd=True,\n",
    "                          creators=[self,other],\n",
    "                          creation_op=\"sub\")\n",
    "        return Tensor(self.data - other.data)\n",
    "    \n",
    "    def __mul__(self, other):\n",
    "        if(self.autograd and other.autograd):\n",
    "            return Tensor(self.data * other.data,\n",
    "                          autograd=True,\n",
    "                          creators=[self,other],\n",
    "                          creation_op=\"mul\")\n",
    "        return Tensor(self.data * other.data)    \n",
    "\n",
    "    def sum(self, dim):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data.sum(dim),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"sum_\"+str(dim))\n",
    "        return Tensor(self.data.sum(dim))\n",
    "    \n",
    "    def expand(self, dim,copies):\n",
    "\n",
    "        trans_cmd = list(range(0,len(self.data.shape)))\n",
    "        trans_cmd.insert(dim,len(self.data.shape))\n",
    "        new_data = self.data.repeat(copies).reshape(list(self.data.shape) + [copies]).transpose(trans_cmd)\n",
    "        \n",
    "        if(self.autograd):\n",
    "            return Tensor(new_data,\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"expand_\"+str(dim))\n",
    "        return Tensor(new_data)\n",
    "    \n",
    "    def transpose(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data.transpose(),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"transpose\")\n",
    "        \n",
    "        return Tensor(self.data.transpose())\n",
    "    \n",
    "    def mm(self, x):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data.dot(x.data),\n",
    "                          autograd=True,\n",
    "                          creators=[self,x],\n",
    "                          creation_op=\"mm\")\n",
    "        return Tensor(self.data.dot(x.data))\n",
    "    \n",
    "    def sigmoid(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(1 / (1 + np.exp(-self.data)),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"sigmoid\")\n",
    "        return Tensor(1 / (1 + np.exp(-self.data)))\n",
    "\n",
    "    def tanh(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(np.tanh(self.data),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"tanh\")\n",
    "        return Tensor(np.tanh(self.data))\n",
    "    \n",
    "    def index_select(self, indices):\n",
    "\n",
    "        if(self.autograd):\n",
    "            new = Tensor(self.data[indices.data],\n",
    "                         autograd=True,\n",
    "                         creators=[self],\n",
    "                         creation_op=\"index_select\")\n",
    "            new.index_select_indices = indices\n",
    "            return new\n",
    "        return Tensor(self.data[indices.data])\n",
    "    \n",
    "    def softmax(self):\n",
    "        temp = np.exp(self.data)\n",
    "        softmax_output = temp / np.sum(temp,\n",
    "                                       axis=len(self.data.shape)-1,\n",
    "                                       keepdims=True)\n",
    "        return softmax_output\n",
    "    \n",
    "    def cross_entropy(self, target_indices):\n",
    "\n",
    "        temp = np.exp(self.data)\n",
    "        softmax_output = temp / np.sum(temp,\n",
    "                                       axis=len(self.data.shape)-1,\n",
    "                                       keepdims=True)\n",
    "        \n",
    "        t = target_indices.data.flatten()\n",
    "        p = softmax_output.reshape(len(t),-1)\n",
    "        target_dist = np.eye(p.shape[1])[t]\n",
    "        loss = -(np.log(p) * (target_dist)).sum(1).mean()\n",
    "    \n",
    "        if(self.autograd):\n",
    "            out = Tensor(loss,\n",
    "                         autograd=True,\n",
    "                         creators=[self],\n",
    "                         creation_op=\"cross_entropy\")\n",
    "            out.softmax_output = softmax_output\n",
    "            out.target_dist = target_dist\n",
    "            return out\n",
    "\n",
    "        return Tensor(loss)\n",
    "        \n",
    "    \n",
    "    def __repr__(self):\n",
    "        return str(self.data.__repr__())\n",
    "    \n",
    "    def __str__(self):\n",
    "        return str(self.data.__str__())  \n",
    "\n",
    "class Layer(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.parameters = list()\n",
    "        \n",
    "    def get_parameters(self):\n",
    "        return self.parameters\n",
    "\n",
    "    \n",
    "class SGD(object):\n",
    "    \n",
    "    def __init__(self, parameters, alpha=0.1):\n",
    "        self.parameters = parameters\n",
    "        self.alpha = alpha\n",
    "    \n",
    "    def zero(self):\n",
    "        for p in self.parameters:\n",
    "            p.grad.data *= 0\n",
    "        \n",
    "    def step(self, zero=True):\n",
    "        \n",
    "        for p in self.parameters:\n",
    "            \n",
    "            p.data -= p.grad.data * self.alpha\n",
    "            \n",
    "            if(zero):\n",
    "                p.grad.data *= 0\n",
    "\n",
    "\n",
    "class Linear(Layer):\n",
    "\n",
    "    def __init__(self, n_inputs, n_outputs, bias=True):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.use_bias = bias\n",
    "        \n",
    "        W = np.random.randn(n_inputs, n_outputs) * np.sqrt(2.0/(n_inputs))\n",
    "        self.weight = Tensor(W, autograd=True)\n",
    "        if(self.use_bias):\n",
    "            self.bias = Tensor(np.zeros(n_outputs), autograd=True)\n",
    "        \n",
    "        self.parameters.append(self.weight)\n",
    "        \n",
    "        if(self.use_bias):        \n",
    "            self.parameters.append(self.bias)\n",
    "\n",
    "    def forward(self, input):\n",
    "        if(self.use_bias):\n",
    "            return input.mm(self.weight)+self.bias.expand(0,len(input.data))\n",
    "        return input.mm(self.weight)\n",
    "\n",
    "\n",
    "class Sequential(Layer):\n",
    "    \n",
    "    def __init__(self, layers=list()):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.layers = layers\n",
    "    \n",
    "    def add(self, layer):\n",
    "        self.layers.append(layer)\n",
    "        \n",
    "    def forward(self, input):\n",
    "        for layer in self.layers:\n",
    "            input = layer.forward(input)\n",
    "        return input\n",
    "    \n",
    "    def get_parameters(self):\n",
    "        params = list()\n",
    "        for l in self.layers:\n",
    "            params += l.get_parameters()\n",
    "        return params\n",
    "\n",
    "\n",
    "class Embedding(Layer):\n",
    "    \n",
    "    def __init__(self, vocab_size, dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.vocab_size = vocab_size\n",
    "        self.dim = dim\n",
    "        \n",
    "        # this random initialiation style is just a convention from word2vec\n",
    "        self.weight = Tensor((np.random.rand(vocab_size, dim) - 0.5) / dim, autograd=True)\n",
    "        \n",
    "        self.parameters.append(self.weight)\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return self.weight.index_select(input)\n",
    "\n",
    "\n",
    "class Tanh(Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return input.tanh()\n",
    "\n",
    "\n",
    "class Sigmoid(Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return input.sigmoid()\n",
    "    \n",
    "\n",
    "class CrossEntropyLoss(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input, target):\n",
    "        return input.cross_entropy(target)\n",
    "\n",
    "    \n",
    "class RNNCell(Layer):\n",
    "    \n",
    "    def __init__(self, n_inputs, n_hidden, n_output, activation='sigmoid'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.n_inputs = n_inputs\n",
    "        self.n_hidden = n_hidden\n",
    "        self.n_output = n_output\n",
    "        \n",
    "        if(activation == 'sigmoid'):\n",
    "            self.activation = Sigmoid()\n",
    "        elif(activation == 'tanh'):\n",
    "            self.activation == Tanh()\n",
    "        else:\n",
    "            raise Exception(\"Non-linearity not found\")\n",
    "\n",
    "        self.w_ih = Linear(n_inputs, n_hidden)\n",
    "        self.w_hh = Linear(n_hidden, n_hidden)\n",
    "        self.w_ho = Linear(n_hidden, n_output)\n",
    "        \n",
    "        self.parameters += self.w_ih.get_parameters()\n",
    "        self.parameters += self.w_hh.get_parameters()\n",
    "        self.parameters += self.w_ho.get_parameters()        \n",
    "    \n",
    "    def forward(self, input, hidden):\n",
    "        from_prev_hidden = self.w_hh.forward(hidden)\n",
    "        combined = self.w_ih.forward(input) + from_prev_hidden\n",
    "        new_hidden = self.activation.forward(combined)\n",
    "        output = self.w_ho.forward(new_hidden)\n",
    "        return output, new_hidden\n",
    "    \n",
    "    def init_hidden(self, batch_size=1):\n",
    "        return Tensor(np.zeros((batch_size,self.n_hidden)), autograd=True)\n",
    "    \n",
    "class LSTMCell(Layer):\n",
    "    \n",
    "    def __init__(self, n_inputs, n_hidden, n_output):\n",
    "        super().__init__()\n",
    "\n",
    "        self.n_inputs = n_inputs\n",
    "        self.n_hidden = n_hidden\n",
    "        self.n_output = n_output\n",
    "\n",
    "        self.xf = Linear(n_inputs, n_hidden)\n",
    "        self.xi = Linear(n_inputs, n_hidden)\n",
    "        self.xo = Linear(n_inputs, n_hidden)        \n",
    "        self.xc = Linear(n_inputs, n_hidden)        \n",
    "        \n",
    "        self.hf = Linear(n_hidden, n_hidden, bias=False)\n",
    "        self.hi = Linear(n_hidden, n_hidden, bias=False)\n",
    "        self.ho = Linear(n_hidden, n_hidden, bias=False)\n",
    "        self.hc = Linear(n_hidden, n_hidden, bias=False)        \n",
    "        \n",
    "        self.w_ho = Linear(n_hidden, n_output, bias=False)\n",
    "        \n",
    "        self.parameters += self.xf.get_parameters()\n",
    "        self.parameters += self.xi.get_parameters()\n",
    "        self.parameters += self.xo.get_parameters()\n",
    "        self.parameters += self.xc.get_parameters()\n",
    "\n",
    "        self.parameters += self.hf.get_parameters()\n",
    "        self.parameters += self.hi.get_parameters()        \n",
    "        self.parameters += self.ho.get_parameters()        \n",
    "        self.parameters += self.hc.get_parameters()                \n",
    "        \n",
    "        self.parameters += self.w_ho.get_parameters()        \n",
    "    \n",
    "    def forward(self, input, hidden):\n",
    "        \n",
    "        prev_hidden = hidden[0]        \n",
    "        prev_cell = hidden[1]\n",
    "        \n",
    "        f = (self.xf.forward(input) + self.hf.forward(prev_hidden)).sigmoid()\n",
    "        i = (self.xi.forward(input) + self.hi.forward(prev_hidden)).sigmoid()\n",
    "        o = (self.xo.forward(input) + self.ho.forward(prev_hidden)).sigmoid()        \n",
    "        g = (self.xc.forward(input) + self.hc.forward(prev_hidden)).tanh()        \n",
    "        c = (f * prev_cell) + (i * g)\n",
    "\n",
    "        h = o * c.tanh()\n",
    "        \n",
    "        output = self.w_ho.forward(h)\n",
    "        return output, (h, c)\n",
    "    \n",
    "    def init_hidden(self, batch_size=1):\n",
    "        init_hidden = Tensor(np.zeros((batch_size,self.n_hidden)), autograd=True)\n",
    "        init_cell = Tensor(np.zeros((batch_size,self.n_hidden)), autograd=True)\n",
    "        init_hidden.data[:,0] += 1\n",
    "        init_cell.data[:,0] += 1\n",
    "        return (init_hidden, init_cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 1: RNN Character Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys,random,math\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import sys\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "# dataset from http://karpathy.github.io/2015/05/21/rnn-effectiveness/\n",
    "f = open('shakespear.txt','r')\n",
    "raw = f.read()\n",
    "f.close()\n",
    "\n",
    "vocab = list(set(raw))\n",
    "word2index = {}\n",
    "for i,word in enumerate(vocab):\n",
    "    word2index[word]=i\n",
    "indices = np.array(list(map(lambda x:word2index[x], raw)))\n",
    "\n",
    "embed = Embedding(vocab_size=len(vocab),dim=512)\n",
    "model = LSTMCell(n_inputs=512, n_hidden=512, n_output=len(vocab))\n",
    "model.w_ho.weight.data *= 0\n",
    "\n",
    "criterion = CrossEntropyLoss()\n",
    "optim = SGD(parameters=model.get_parameters() + embed.get_parameters(), alpha=0.05)\n",
    "\n",
    "def generate_sample(n=30, init_char=' '):\n",
    "    s = \"\"\n",
    "    hidden = model.init_hidden(batch_size=1)\n",
    "    input = Tensor(np.array([word2index[init_char]]))\n",
    "    for i in range(n):\n",
    "        rnn_input = embed.forward(input)\n",
    "        output, hidden = model.forward(input=rnn_input, hidden=hidden)\n",
    "#         output.data *= 25\n",
    "#         temp_dist = output.softmax()\n",
    "#         temp_dist /= temp_dist.sum()\n",
    "\n",
    "#         m = (temp_dist > np.random.rand()).argmax()\n",
    "        m = output.data.argmax()\n",
    "        c = vocab[m]\n",
    "        input = Tensor(np.array([m]))\n",
    "        s += c\n",
    "    return s\n",
    "\n",
    "batch_size = 16\n",
    "bptt = 25\n",
    "n_batches = int((indices.shape[0] / (batch_size)))\n",
    "\n",
    "trimmed_indices = indices[:n_batches*batch_size]\n",
    "batched_indices = trimmed_indices.reshape(batch_size, n_batches).transpose()\n",
    "\n",
    "input_batched_indices = batched_indices[0:-1]\n",
    "target_batched_indices = batched_indices[1:]\n",
    "\n",
    "n_bptt = int(((n_batches-1) / bptt))\n",
    "input_batches = input_batched_indices[:n_bptt*bptt].reshape(n_bptt,bptt,batch_size)\n",
    "target_batches = target_batched_indices[:n_bptt*bptt].reshape(n_bptt, bptt, batch_size)\n",
    "min_loss = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(iterations=400):\n",
    "    for iter in range(iterations):\n",
    "        total_loss = 0\n",
    "        n_loss = 0\n",
    "\n",
    "        hidden = model.init_hidden(batch_size=batch_size)\n",
    "        batches_to_train = len(input_batches)\n",
    "    #     batches_to_train = 32\n",
    "        for batch_i in range(batches_to_train):\n",
    "\n",
    "            hidden = (Tensor(hidden[0].data, autograd=True), Tensor(hidden[1].data, autograd=True))\n",
    "\n",
    "            losses = list()\n",
    "            for t in range(bptt):\n",
    "                input = Tensor(input_batches[batch_i][t], autograd=True)\n",
    "                rnn_input = embed.forward(input=input)\n",
    "                output, hidden = model.forward(input=rnn_input, hidden=hidden)\n",
    "\n",
    "                target = Tensor(target_batches[batch_i][t], autograd=True)    \n",
    "                batch_loss = criterion.forward(output, target)\n",
    "\n",
    "                if(t == 0):\n",
    "                    losses.append(batch_loss)\n",
    "                else:\n",
    "                    losses.append(batch_loss + losses[-1])\n",
    "\n",
    "            loss = losses[-1]\n",
    "\n",
    "            loss.backward()\n",
    "            optim.step()\n",
    "            total_loss += loss.data / bptt\n",
    "\n",
    "            epoch_loss = np.exp(total_loss / (batch_i+1))\n",
    "            if(epoch_loss < min_loss):\n",
    "                min_loss = epoch_loss\n",
    "                print()\n",
    "\n",
    "            log = \"\\r Iter:\" + str(iter)\n",
    "            log += \" - Alpha:\" + str(optim.alpha)[0:5]\n",
    "            log += \" - Batch \"+str(batch_i+1)+\"/\"+str(len(input_batches))\n",
    "            log += \" - Min Loss:\" + str(min_loss)[0:5]\n",
    "            log += \" - Loss:\" + str(epoch_loss)\n",
    "            if(batch_i == 0):\n",
    "                log += \" - \" + generate_sample(n=70, init_char='T').replace(\"\\n\",\" \")\n",
    "            if(batch_i % 1 == 0):\n",
    "                sys.stdout.write(log)\n",
    "        optim.alpha *= 0.99\n",
    "    #     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Iter:0 - Alpha:0.05 - Batch 1/249 - Min Loss:62.00 - Loss:62.000000000000064 -           eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\n",
      " Iter:0 - Alpha:0.05 - Batch 2/249 - Min Loss:61.99 - Loss:61.999336055802885\n",
      " Iter:0 - Alpha:0.05 - Batch 3/249 - Min Loss:61.98 - Loss:61.989983546689196\n",
      " Iter:0 - Alpha:0.05 - Batch 4/249 - Min Loss:61.97 - Loss:61.972948235165255\n",
      " Iter:0 - Alpha:0.05 - Batch 5/249 - Min Loss:61.94 - Loss:61.941383549495384\n",
      " Iter:0 - Alpha:0.05 - Batch 6/249 - Min Loss:61.88 - Loss:61.88023671827271\n",
      " Iter:0 - Alpha:0.05 - Batch 7/249 - Min Loss:61.77 - Loss:61.77690827437837\n",
      " Iter:0 - Alpha:0.05 - Batch 8/249 - Min Loss:61.52 - Loss:61.52953899883961\n",
      " Iter:0 - Alpha:0.05 - Batch 9/249 - Min Loss:61.00 - Loss:61.00486153547285\n",
      " Iter:0 - Alpha:0.05 - Batch 10/249 - Min Loss:60.23 - Loss:60.236912186726684\n",
      " Iter:0 - Alpha:0.05 - Batch 11/249 - Min Loss:58.70 - Loss:58.7055559369767\n",
      " Iter:0 - Alpha:0.05 - Batch 12/249 - Min Loss:56.73 - Loss:56.73775220158473\n",
      " Iter:0 - Alpha:0.05 - Batch 13/249 - Min Loss:54.10 - Loss:54.10996106485584\n",
      " Iter:0 - Alpha:0.05 - Batch 14/249 - Min Loss:52.75 - Loss:52.75637293050057\n",
      " Iter:0 - Alpha:0.05 - Batch 15/249 - Min Loss:51.07 - Loss:51.078681882080105\n",
      " Iter:0 - Alpha:0.05 - Batch 16/249 - Min Loss:49.37 - Loss:49.37743406427449\n",
      " Iter:0 - Alpha:0.05 - Batch 17/249 - Min Loss:47.81 - Loss:47.81006661764188\n",
      " Iter:0 - Alpha:0.05 - Batch 18/249 - Min Loss:46.68 - Loss:46.68131330399904\n",
      " Iter:0 - Alpha:0.05 - Batch 19/249 - Min Loss:45.76 - Loss:45.76135529411921\n",
      " Iter:0 - Alpha:0.05 - Batch 20/249 - Min Loss:44.63 - Loss:44.63742967139992\n",
      " Iter:0 - Alpha:0.05 - Batch 21/249 - Min Loss:43.43 - Loss:43.43315342999167\n",
      " Iter:0 - Alpha:0.05 - Batch 22/249 - Min Loss:43.13 - Loss:43.133727315170454\n",
      " Iter:0 - Alpha:0.05 - Batch 23/249 - Min Loss:43.08 - Loss:43.08924458053491\n",
      " Iter:0 - Alpha:0.05 - Batch 24/249 - Min Loss:42.48 - Loss:42.48625785761426\n",
      " Iter:0 - Alpha:0.05 - Batch 25/249 - Min Loss:41.59 - Loss:41.59564764008973\n",
      " Iter:0 - Alpha:0.05 - Batch 26/249 - Min Loss:40.64 - Loss:40.64633262212879\n",
      " Iter:0 - Alpha:0.05 - Batch 27/249 - Min Loss:40.08 - Loss:40.08437857978491\n",
      " Iter:0 - Alpha:0.05 - Batch 28/249 - Min Loss:39.38 - Loss:39.38197568983813\n",
      " Iter:0 - Alpha:0.05 - Batch 29/249 - Min Loss:38.85 - Loss:38.85036603038319\n",
      " Iter:0 - Alpha:0.05 - Batch 30/249 - Min Loss:38.32 - Loss:38.32050246588233\n",
      " Iter:0 - Alpha:0.05 - Batch 31/249 - Min Loss:38.02 - Loss:38.028742643067304\n",
      " Iter:0 - Alpha:0.05 - Batch 32/249 - Min Loss:37.57 - Loss:37.579230715808585\n",
      " Iter:0 - Alpha:0.05 - Batch 33/249 - Min Loss:37.15 - Loss:37.1513332533316\n",
      " Iter:0 - Alpha:0.05 - Batch 34/249 - Min Loss:36.72 - Loss:36.72716819545398\n",
      " Iter:0 - Alpha:0.05 - Batch 35/249 - Min Loss:36.50 - Loss:36.505523013835905\n",
      " Iter:0 - Alpha:0.05 - Batch 36/249 - Min Loss:36.26 - Loss:36.264791172196766\n",
      " Iter:0 - Alpha:0.05 - Batch 37/249 - Min Loss:35.93 - Loss:35.93241785924657\n",
      " Iter:0 - Alpha:0.05 - Batch 39/249 - Min Loss:35.69 - Loss:35.69162009265761\n",
      " Iter:0 - Alpha:0.05 - Batch 40/249 - Min Loss:35.39 - Loss:35.391766263709975\n",
      " Iter:0 - Alpha:0.05 - Batch 41/249 - Min Loss:35.24 - Loss:35.24024995428248\n",
      " Iter:0 - Alpha:0.05 - Batch 42/249 - Min Loss:35.16 - Loss:35.16636943540858\n",
      " Iter:0 - Alpha:0.05 - Batch 43/249 - Min Loss:34.82 - Loss:34.82115954562641\n",
      " Iter:0 - Alpha:0.05 - Batch 44/249 - Min Loss:34.60 - Loss:34.60065020684661\n",
      " Iter:0 - Alpha:0.05 - Batch 45/249 - Min Loss:34.54 - Loss:34.549754104397785\n",
      " Iter:0 - Alpha:0.05 - Batch 46/249 - Min Loss:34.32 - Loss:34.32999305867251\n",
      " Iter:0 - Alpha:0.05 - Batch 47/249 - Min Loss:34.11 - Loss:34.117257032452486\n",
      " Iter:0 - Alpha:0.05 - Batch 48/249 - Min Loss:33.90 - Loss:33.90487349892798\n",
      " Iter:0 - Alpha:0.05 - Batch 49/249 - Min Loss:33.75 - Loss:33.75966234624244\n",
      " Iter:0 - Alpha:0.05 - Batch 50/249 - Min Loss:33.61 - Loss:33.61016131907992\n",
      " Iter:0 - Alpha:0.05 - Batch 51/249 - Min Loss:33.31 - Loss:33.31167842936299\n",
      " Iter:0 - Alpha:0.05 - Batch 52/249 - Min Loss:33.11 - Loss:33.11904817623289\n",
      " Iter:0 - Alpha:0.05 - Batch 53/249 - Min Loss:33.00 - Loss:33.004345599234625\n",
      " Iter:0 - Alpha:0.05 - Batch 54/249 - Min Loss:32.81 - Loss:32.817426265009786\n",
      " Iter:0 - Alpha:0.05 - Batch 55/249 - Min Loss:32.60 - Loss:32.60854105028041\n",
      " Iter:0 - Alpha:0.05 - Batch 56/249 - Min Loss:32.42 - Loss:32.42050431188535\n",
      " Iter:0 - Alpha:0.05 - Batch 57/249 - Min Loss:32.19 - Loss:32.198334721563576\n",
      " Iter:0 - Alpha:0.05 - Batch 58/249 - Min Loss:32.02 - Loss:32.027256645802886\n",
      " Iter:0 - Alpha:0.05 - Batch 59/249 - Min Loss:31.81 - Loss:31.818082530542316\n",
      " Iter:0 - Alpha:0.05 - Batch 60/249 - Min Loss:31.63 - Loss:31.631415472551268\n",
      " Iter:0 - Alpha:0.05 - Batch 61/249 - Min Loss:31.39 - Loss:31.393959746851287\n",
      " Iter:0 - Alpha:0.05 - Batch 62/249 - Min Loss:31.19 - Loss:31.19942305047541\n",
      " Iter:0 - Alpha:0.05 - Batch 63/249 - Min Loss:30.95 - Loss:30.95343987653838\n",
      " Iter:0 - Alpha:0.05 - Batch 64/249 - Min Loss:30.74 - Loss:30.7489265136333\n",
      " Iter:0 - Alpha:0.05 - Batch 65/249 - Min Loss:30.66 - Loss:30.665132078887083\n",
      " Iter:0 - Alpha:0.05 - Batch 66/249 - Min Loss:30.60 - Loss:30.605053974596405\n",
      " Iter:0 - Alpha:0.05 - Batch 67/249 - Min Loss:30.45 - Loss:30.456479778353064\n",
      " Iter:0 - Alpha:0.05 - Batch 68/249 - Min Loss:30.24 - Loss:30.241772045820696\n",
      " Iter:0 - Alpha:0.05 - Batch 69/249 - Min Loss:30.14 - Loss:30.14494883095973\n",
      " Iter:0 - Alpha:0.05 - Batch 70/249 - Min Loss:30.00 - Loss:30.004543678506863\n",
      " Iter:0 - Alpha:0.05 - Batch 71/249 - Min Loss:29.89 - Loss:29.899002156894124\n",
      " Iter:0 - Alpha:0.05 - Batch 72/249 - Min Loss:29.85 - Loss:29.85125401656389\n",
      " Iter:0 - Alpha:0.05 - Batch 73/249 - Min Loss:29.74 - Loss:29.742558662511755\n",
      " Iter:0 - Alpha:0.05 - Batch 74/249 - Min Loss:29.55 - Loss:29.554013392531395\n",
      " Iter:0 - Alpha:0.05 - Batch 75/249 - Min Loss:29.41 - Loss:29.413451221065877\n",
      " Iter:0 - Alpha:0.05 - Batch 76/249 - Min Loss:29.30 - Loss:29.300279999145584\n",
      " Iter:0 - Alpha:0.05 - Batch 77/249 - Min Loss:29.16 - Loss:29.162816380218032\n",
      " Iter:0 - Alpha:0.05 - Batch 78/249 - Min Loss:29.09 - Loss:29.09592033129947\n",
      " Iter:0 - Alpha:0.05 - Batch 79/249 - Min Loss:28.96 - Loss:28.969066622778954\n",
      " Iter:0 - Alpha:0.05 - Batch 80/249 - Min Loss:28.84 - Loss:28.847024497772598\n",
      " Iter:0 - Alpha:0.05 - Batch 81/249 - Min Loss:28.68 - Loss:28.682792440948468\n",
      " Iter:0 - Alpha:0.05 - Batch 82/249 - Min Loss:28.59 - Loss:28.598158009843733\n",
      " Iter:0 - Alpha:0.05 - Batch 86/249 - Min Loss:28.54 - Loss:28.672844929166207\n",
      " Iter:0 - Alpha:0.05 - Batch 87/249 - Min Loss:28.50 - Loss:28.5084865620548\n",
      " Iter:0 - Alpha:0.05 - Batch 88/249 - Min Loss:28.35 - Loss:28.352102311265327\n",
      " Iter:0 - Alpha:0.05 - Batch 89/249 - Min Loss:28.19 - Loss:28.192980003869685\n",
      " Iter:0 - Alpha:0.05 - Batch 90/249 - Min Loss:28.05 - Loss:28.052399362314123\n",
      " Iter:0 - Alpha:0.05 - Batch 91/249 - Min Loss:27.92 - Loss:27.928541406577803\n",
      " Iter:0 - Alpha:0.05 - Batch 92/249 - Min Loss:27.82 - Loss:27.826934205543306\n",
      " Iter:0 - Alpha:0.05 - Batch 93/249 - Min Loss:27.67 - Loss:27.67999675592469\n",
      " Iter:0 - Alpha:0.05 - Batch 94/249 - Min Loss:27.54 - Loss:27.549420657967516\n",
      " Iter:0 - Alpha:0.05 - Batch 95/249 - Min Loss:27.40 - Loss:27.40841828220945\n",
      " Iter:0 - Alpha:0.05 - Batch 96/249 - Min Loss:27.31 - Loss:27.318381171485257\n",
      " Iter:0 - Alpha:0.05 - Batch 97/249 - Min Loss:27.19 - Loss:27.199553368220013\n",
      " Iter:0 - Alpha:0.05 - Batch 98/249 - Min Loss:27.08 - Loss:27.085005756069428\n",
      " Iter:0 - Alpha:0.05 - Batch 99/249 - Min Loss:26.92 - Loss:26.920652799695258\n",
      " Iter:0 - Alpha:0.05 - Batch 100/249 - Min Loss:26.77 - Loss:26.778338171170603\n",
      " Iter:0 - Alpha:0.05 - Batch 101/249 - Min Loss:26.63 - Loss:26.63444492542303\n",
      " Iter:0 - Alpha:0.05 - Batch 102/249 - Min Loss:26.50 - Loss:26.503214805264342\n",
      " Iter:0 - Alpha:0.05 - Batch 103/249 - Min Loss:26.38 - Loss:26.38627303021265\n",
      " Iter:0 - Alpha:0.05 - Batch 104/249 - Min Loss:26.28 - Loss:26.28984072160501\n",
      " Iter:0 - Alpha:0.05 - Batch 105/249 - Min Loss:26.19 - Loss:26.197356511158755\n",
      " Iter:0 - Alpha:0.05 - Batch 106/249 - Min Loss:26.11 - Loss:26.110568826540085\n",
      " Iter:0 - Alpha:0.05 - Batch 107/249 - Min Loss:26.08 - Loss:26.08702025964796\n",
      " Iter:0 - Alpha:0.05 - Batch 108/249 - Min Loss:25.98 - Loss:25.98179026707607\n",
      " Iter:0 - Alpha:0.05 - Batch 109/249 - Min Loss:25.89 - Loss:25.891771314535525\n",
      " Iter:0 - Alpha:0.05 - Batch 110/249 - Min Loss:25.80 - Loss:25.802742883596974\n",
      " Iter:0 - Alpha:0.05 - Batch 111/249 - Min Loss:25.71 - Loss:25.717937269876025\n",
      " Iter:0 - Alpha:0.05 - Batch 112/249 - Min Loss:25.64 - Loss:25.641765977541738\n",
      " Iter:0 - Alpha:0.05 - Batch 113/249 - Min Loss:25.57 - Loss:25.57325789739905\n",
      " Iter:0 - Alpha:0.05 - Batch 114/249 - Min Loss:25.49 - Loss:25.499436282313912\n",
      " Iter:0 - Alpha:0.05 - Batch 115/249 - Min Loss:25.44 - Loss:25.442407943785195\n",
      " Iter:0 - Alpha:0.05 - Batch 116/249 - Min Loss:25.35 - Loss:25.35542832928056\n",
      " Iter:0 - Alpha:0.05 - Batch 117/249 - Min Loss:25.27 - Loss:25.276678269744394\n",
      " Iter:0 - Alpha:0.05 - Batch 118/249 - Min Loss:25.17 - Loss:25.175983940560986\n",
      " Iter:0 - Alpha:0.05 - Batch 119/249 - Min Loss:25.10 - Loss:25.10534037752\n",
      " Iter:0 - Alpha:0.05 - Batch 120/249 - Min Loss:25.01 - Loss:25.010115025991002\n",
      " Iter:0 - Alpha:0.05 - Batch 121/249 - Min Loss:24.92 - Loss:24.920317590856904\n",
      " Iter:0 - Alpha:0.05 - Batch 122/249 - Min Loss:24.82 - Loss:24.822277409798907\n",
      " Iter:0 - Alpha:0.05 - Batch 123/249 - Min Loss:24.72 - Loss:24.72203946090018\n",
      " Iter:0 - Alpha:0.05 - Batch 124/249 - Min Loss:24.62 - Loss:24.626004292971295\n",
      " Iter:0 - Alpha:0.05 - Batch 125/249 - Min Loss:24.54 - Loss:24.548408835881084\n",
      " Iter:0 - Alpha:0.05 - Batch 126/249 - Min Loss:24.48 - Loss:24.481874816478964\n",
      " Iter:0 - Alpha:0.05 - Batch 127/249 - Min Loss:24.39 - Loss:24.397815037786064\n",
      " Iter:0 - Alpha:0.05 - Batch 128/249 - Min Loss:24.29 - Loss:24.29141790917974\n",
      " Iter:0 - Alpha:0.05 - Batch 129/249 - Min Loss:24.19 - Loss:24.19678204070675\n",
      " Iter:0 - Alpha:0.05 - Batch 130/249 - Min Loss:24.12 - Loss:24.126267171636325\n",
      " Iter:0 - Alpha:0.05 - Batch 131/249 - Min Loss:24.03 - Loss:24.031656923161755\n",
      " Iter:0 - Alpha:0.05 - Batch 132/249 - Min Loss:23.93 - Loss:23.93408720034178\n",
      " Iter:0 - Alpha:0.05 - Batch 133/249 - Min Loss:23.85 - Loss:23.850310797547163\n",
      " Iter:0 - Alpha:0.05 - Batch 134/249 - Min Loss:23.76 - Loss:23.76517754738187\n",
      " Iter:0 - Alpha:0.05 - Batch 135/249 - Min Loss:23.71 - Loss:23.715494372742555\n",
      " Iter:0 - Alpha:0.05 - Batch 136/249 - Min Loss:23.70 - Loss:23.701413122242627\n",
      " Iter:0 - Alpha:0.05 - Batch 137/249 - Min Loss:23.62 - Loss:23.628566884474214\n",
      " Iter:0 - Alpha:0.05 - Batch 138/249 - Min Loss:23.53 - Loss:23.536622255870185\n",
      " Iter:0 - Alpha:0.05 - Batch 140/249 - Min Loss:23.51 - Loss:23.553200160956592\n",
      " Iter:0 - Alpha:0.05 - Batch 141/249 - Min Loss:23.49 - Loss:23.4954391983157\n",
      " Iter:0 - Alpha:0.05 - Batch 142/249 - Min Loss:23.40 - Loss:23.407151066893466\n",
      " Iter:0 - Alpha:0.05 - Batch 143/249 - Min Loss:23.32 - Loss:23.322406347920353\n",
      " Iter:0 - Alpha:0.05 - Batch 144/249 - Min Loss:23.23 - Loss:23.23392038507518\n",
      " Iter:0 - Alpha:0.05 - Batch 145/249 - Min Loss:23.15 - Loss:23.158962886202442\n",
      " Iter:0 - Alpha:0.05 - Batch 146/249 - Min Loss:23.13 - Loss:23.13027389905198\n",
      " Iter:0 - Alpha:0.05 - Batch 147/249 - Min Loss:23.08 - Loss:23.08432984727065\n",
      " Iter:0 - Alpha:0.05 - Batch 148/249 - Min Loss:23.05 - Loss:23.052812278291672\n",
      " Iter:0 - Alpha:0.05 - Batch 149/249 - Min Loss:22.97 - Loss:22.977617993036645\n",
      " Iter:0 - Alpha:0.05 - Batch 150/249 - Min Loss:22.90 - Loss:22.90860982682122\n",
      " Iter:0 - Alpha:0.05 - Batch 151/249 - Min Loss:22.86 - Loss:22.86285029247915\n",
      " Iter:0 - Alpha:0.05 - Batch 152/249 - Min Loss:22.79 - Loss:22.798324855724506\n",
      " Iter:0 - Alpha:0.05 - Batch 153/249 - Min Loss:22.71 - Loss:22.714156980919036\n",
      " Iter:0 - Alpha:0.05 - Batch 154/249 - Min Loss:22.64 - Loss:22.649942295215556\n",
      " Iter:0 - Alpha:0.05 - Batch 155/249 - Min Loss:22.60 - Loss:22.60987787211344\n",
      " Iter:0 - Alpha:0.05 - Batch 156/249 - Min Loss:22.58 - Loss:22.586330875896408\n",
      " Iter:0 - Alpha:0.05 - Batch 157/249 - Min Loss:22.53 - Loss:22.539866221935924\n",
      " Iter:0 - Alpha:0.05 - Batch 158/249 - Min Loss:22.48 - Loss:22.481459540414978\n",
      " Iter:0 - Alpha:0.05 - Batch 159/249 - Min Loss:22.44 - Loss:22.44458654448001\n",
      " Iter:0 - Alpha:0.05 - Batch 160/249 - Min Loss:22.38 - Loss:22.383503615633796\n",
      " Iter:0 - Alpha:0.05 - Batch 161/249 - Min Loss:22.31 - Loss:22.317076639885897\n",
      " Iter:0 - Alpha:0.05 - Batch 162/249 - Min Loss:22.25 - Loss:22.259430027034902\n",
      " Iter:0 - Alpha:0.05 - Batch 163/249 - Min Loss:22.19 - Loss:22.19904509067437\n",
      " Iter:0 - Alpha:0.05 - Batch 164/249 - Min Loss:22.14 - Loss:22.148098444656362\n",
      " Iter:0 - Alpha:0.05 - Batch 165/249 - Min Loss:22.10 - Loss:22.101528454497767\n",
      " Iter:0 - Alpha:0.05 - Batch 166/249 - Min Loss:22.04 - Loss:22.04778889335714\n",
      " Iter:0 - Alpha:0.05 - Batch 167/249 - Min Loss:22.01 - Loss:22.015539256206182\n",
      " Iter:0 - Alpha:0.05 - Batch 168/249 - Min Loss:21.97 - Loss:21.97610738320577\n",
      " Iter:0 - Alpha:0.05 - Batch 169/249 - Min Loss:21.91 - Loss:21.91886403464624\n",
      " Iter:0 - Alpha:0.05 - Batch 170/249 - Min Loss:21.87 - Loss:21.87209957981593\n",
      " Iter:0 - Alpha:0.05 - Batch 171/249 - Min Loss:21.83 - Loss:21.83197231706187\n",
      " Iter:0 - Alpha:0.05 - Batch 172/249 - Min Loss:21.79 - Loss:21.797138291435964\n",
      " Iter:0 - Alpha:0.05 - Batch 173/249 - Min Loss:21.75 - Loss:21.752929637839053\n",
      " Iter:0 - Alpha:0.05 - Batch 174/249 - Min Loss:21.71 - Loss:21.71451452173225\n",
      " Iter:0 - Alpha:0.05 - Batch 175/249 - Min Loss:21.68 - Loss:21.68635181976672\n",
      " Iter:0 - Alpha:0.05 - Batch 176/249 - Min Loss:21.64 - Loss:21.643368363210932\n",
      " Iter:0 - Alpha:0.05 - Batch 177/249 - Min Loss:21.59 - Loss:21.59547234826887\n",
      " Iter:0 - Alpha:0.05 - Batch 178/249 - Min Loss:21.54 - Loss:21.548969384961\n",
      " Iter:0 - Alpha:0.05 - Batch 179/249 - Min Loss:21.52 - Loss:21.528790503616882\n",
      " Iter:0 - Alpha:0.05 - Batch 180/249 - Min Loss:21.48 - Loss:21.484217083500187\n",
      " Iter:0 - Alpha:0.05 - Batch 181/249 - Min Loss:21.43 - Loss:21.43585185733848\n",
      " Iter:0 - Alpha:0.05 - Batch 182/249 - Min Loss:21.38 - Loss:21.386494316884\n",
      " Iter:0 - Alpha:0.05 - Batch 183/249 - Min Loss:21.33 - Loss:21.333293821976987\n",
      " Iter:0 - Alpha:0.05 - Batch 184/249 - Min Loss:21.29 - Loss:21.292714546692963\n",
      " Iter:0 - Alpha:0.05 - Batch 185/249 - Min Loss:21.24 - Loss:21.2479883334045\n",
      " Iter:0 - Alpha:0.05 - Batch 186/249 - Min Loss:21.21 - Loss:21.21454412091733\n",
      " Iter:0 - Alpha:0.05 - Batch 187/249 - Min Loss:21.18 - Loss:21.18709730607852\n",
      " Iter:0 - Alpha:0.05 - Batch 188/249 - Min Loss:21.18 - Loss:21.18500879669473\n",
      " Iter:0 - Alpha:0.05 - Batch 189/249 - Min Loss:21.14 - Loss:21.149192169258725\n",
      " Iter:0 - Alpha:0.05 - Batch 190/249 - Min Loss:21.12 - Loss:21.121492290739525\n",
      " Iter:0 - Alpha:0.05 - Batch 191/249 - Min Loss:21.09 - Loss:21.090529064322375\n",
      " Iter:0 - Alpha:0.05 - Batch 192/249 - Min Loss:21.06 - Loss:21.061615675909103\n",
      " Iter:0 - Alpha:0.05 - Batch 193/249 - Min Loss:21.00 - Loss:21.000099027990963\n",
      " Iter:0 - Alpha:0.05 - Batch 194/249 - Min Loss:20.98 - Loss:20.98468231590407\n",
      " Iter:0 - Alpha:0.05 - Batch 195/249 - Min Loss:20.98 - Loss:20.983618463178825\n",
      " Iter:0 - Alpha:0.05 - Batch 196/249 - Min Loss:20.94 - Loss:20.941073976275163\n",
      " Iter:0 - Alpha:0.05 - Batch 197/249 - Min Loss:20.90 - Loss:20.909982209463195\n",
      " Iter:0 - Alpha:0.05 - Batch 198/249 - Min Loss:20.87 - Loss:20.8792320482417\n",
      " Iter:0 - Alpha:0.05 - Batch 199/249 - Min Loss:20.83 - Loss:20.836144396820785\n",
      " Iter:0 - Alpha:0.05 - Batch 200/249 - Min Loss:20.78 - Loss:20.786702940643394\n",
      " Iter:0 - Alpha:0.05 - Batch 201/249 - Min Loss:20.75 - Loss:20.755678930297545\n",
      " Iter:0 - Alpha:0.05 - Batch 202/249 - Min Loss:20.72 - Loss:20.72621131450987\n",
      " Iter:0 - Alpha:0.05 - Batch 203/249 - Min Loss:20.68 - Loss:20.68893807384813\n",
      " Iter:0 - Alpha:0.05 - Batch 204/249 - Min Loss:20.65 - Loss:20.650964182346172\n",
      " Iter:0 - Alpha:0.05 - Batch 205/249 - Min Loss:20.61 - Loss:20.617254141780762\n",
      " Iter:0 - Alpha:0.05 - Batch 206/249 - Min Loss:20.57 - Loss:20.573633574910954\n",
      " Iter:0 - Alpha:0.05 - Batch 207/249 - Min Loss:20.53 - Loss:20.53542759736034\n",
      " Iter:0 - Alpha:0.05 - Batch 208/249 - Min Loss:20.49 - Loss:20.49739996174571\n",
      " Iter:0 - Alpha:0.05 - Batch 209/249 - Min Loss:20.44 - Loss:20.449686230569824\n",
      " Iter:0 - Alpha:0.05 - Batch 210/249 - Min Loss:20.40 - Loss:20.403291221330196\n",
      " Iter:0 - Alpha:0.05 - Batch 211/249 - Min Loss:20.34 - Loss:20.3475100925815\n",
      " Iter:0 - Alpha:0.05 - Batch 212/249 - Min Loss:20.30 - Loss:20.301789206440752\n",
      " Iter:0 - Alpha:0.05 - Batch 213/249 - Min Loss:20.26 - Loss:20.26542625947414\n",
      " Iter:0 - Alpha:0.05 - Batch 214/249 - Min Loss:20.23 - Loss:20.23111001746059\n",
      " Iter:0 - Alpha:0.05 - Batch 215/249 - Min Loss:20.18 - Loss:20.18481394544832\n",
      " Iter:0 - Alpha:0.05 - Batch 216/249 - Min Loss:20.14 - Loss:20.144828981387178\n",
      " Iter:0 - Alpha:0.05 - Batch 217/249 - Min Loss:20.11 - Loss:20.117436374376467\n",
      " Iter:0 - Alpha:0.05 - Batch 218/249 - Min Loss:20.07 - Loss:20.07285790792156\n",
      " Iter:0 - Alpha:0.05 - Batch 219/249 - Min Loss:20.01 - Loss:20.019440903545878\n",
      " Iter:0 - Alpha:0.05 - Batch 220/249 - Min Loss:19.99 - Loss:19.995961465663378\n",
      " Iter:0 - Alpha:0.05 - Batch 221/249 - Min Loss:19.98 - Loss:19.986751087464363\n",
      " Iter:0 - Alpha:0.05 - Batch 222/249 - Min Loss:19.95 - Loss:19.95413004376577\n",
      " Iter:0 - Alpha:0.05 - Batch 223/249 - Min Loss:19.92 - Loss:19.920535628027963\n",
      " Iter:0 - Alpha:0.05 - Batch 224/249 - Min Loss:19.88 - Loss:19.88982739248521\n",
      " Iter:0 - Alpha:0.05 - Batch 225/249 - Min Loss:19.85 - Loss:19.85661224385163\n",
      " Iter:0 - Alpha:0.05 - Batch 226/249 - Min Loss:19.81 - Loss:19.818270498440558\n",
      " Iter:0 - Alpha:0.05 - Batch 227/249 - Min Loss:19.78 - Loss:19.78879746898553\n",
      " Iter:0 - Alpha:0.05 - Batch 228/249 - Min Loss:19.77 - Loss:19.773987163273635\n",
      " Iter:0 - Alpha:0.05 - Batch 229/249 - Min Loss:19.74 - Loss:19.74321893462951\n",
      " Iter:0 - Alpha:0.05 - Batch 230/249 - Min Loss:19.71 - Loss:19.714114028548735\n",
      " Iter:0 - Alpha:0.05 - Batch 231/249 - Min Loss:19.69 - Loss:19.696300464252033\n",
      " Iter:0 - Alpha:0.05 - Batch 232/249 - Min Loss:19.66 - Loss:19.664281554171573\n",
      " Iter:0 - Alpha:0.05 - Batch 233/249 - Min Loss:19.64 - Loss:19.64497900030178\n",
      " Iter:0 - Alpha:0.05 - Batch 234/249 - Min Loss:19.61 - Loss:19.61390018795743\n",
      " Iter:0 - Alpha:0.05 - Batch 235/249 - Min Loss:19.57 - Loss:19.578057993323174\n",
      " Iter:0 - Alpha:0.05 - Batch 236/249 - Min Loss:19.53 - Loss:19.539290922887076\n",
      " Iter:0 - Alpha:0.05 - Batch 237/249 - Min Loss:19.51 - Loss:19.51336152702854\n",
      " Iter:0 - Alpha:0.05 - Batch 238/249 - Min Loss:19.49 - Loss:19.494216997205427\n",
      " Iter:0 - Alpha:0.05 - Batch 239/249 - Min Loss:19.47 - Loss:19.474255434858627\n",
      " Iter:0 - Alpha:0.05 - Batch 240/249 - Min Loss:19.45 - Loss:19.45452512691269\n",
      " Iter:0 - Alpha:0.05 - Batch 241/249 - Min Loss:19.41 - Loss:19.41567509465724\n",
      " Iter:0 - Alpha:0.05 - Batch 242/249 - Min Loss:19.38 - Loss:19.384809146622928\n",
      " Iter:0 - Alpha:0.05 - Batch 244/249 - Min Loss:19.35 - Loss:19.35866317796798\n",
      " Iter:0 - Alpha:0.05 - Batch 245/249 - Min Loss:19.33 - Loss:19.33615864113177\n",
      " Iter:0 - Alpha:0.05 - Batch 246/249 - Min Loss:19.31 - Loss:19.311764056907894\n",
      " Iter:0 - Alpha:0.05 - Batch 247/249 - Min Loss:19.28 - Loss:19.28639653162862\n",
      " Iter:0 - Alpha:0.05 - Batch 248/249 - Min Loss:19.25 - Loss:19.259355873841894\n",
      " Iter:0 - Alpha:0.05 - Batch 249/249 - Min Loss:19.23 - Loss:19.233214769358476\n",
      " Iter:1 - Alpha:0.049 - Batch 1/249 - Min Loss:13.06 - Loss:13.063830116471486 - her t tere t tere t tere t tere t tere t tere t tere t tere t tere t t\n",
      " Iter:1 - Alpha:0.049 - Batch 3/249 - Min Loss:12.94 - Loss:13.045405787590937\n",
      " Iter:1 - Alpha:0.049 - Batch 4/249 - Min Loss:12.93 - Loss:12.931715871054474\n",
      " Iter:1 - Alpha:0.049 - Batch 249/249 - Min Loss:12.88 - Loss:13.297767268945559\n",
      " Iter:2 - Alpha:0.049 - Batch 249/249 - Min Loss:11.98 - Loss:12.467682706214898 hen theren theren therer then theren theren therer then theren theren \n",
      " Iter:3 - Alpha:0.048 - Batch 2/249 - Min Loss:11.43 - Loss:11.463986229895673 - hen theres thes thes thes thes thes thes thes thes thes thes thes thes\n",
      " Iter:3 - Alpha:0.048 - Batch 3/249 - Min Loss:11.43 - Loss:11.433608994379455\n",
      " Iter:3 - Alpha:0.048 - Batch 4/249 - Min Loss:11.29 - Loss:11.292592685693808\n",
      " Iter:5 - Alpha:0.047 - Batch 55/249 - Min Loss:11.19 - Loss:11.211233778075991- hend seates, and seates, and seates, and seates, and seates, and seate\n",
      " Iter:5 - Alpha:0.047 - Batch 56/249 - Min Loss:11.17 - Loss:11.172951972803446\n",
      " Iter:5 - Alpha:0.047 - Batch 100/249 - Min Loss:11.14 - Loss:11.15109606209378\n",
      " Iter:5 - Alpha:0.047 - Batch 101/249 - Min Loss:11.13 - Loss:11.13802901054302\n",
      " Iter:5 - Alpha:0.047 - Batch 102/249 - Min Loss:11.11 - Loss:11.119285037049702\n",
      " Iter:5 - Alpha:0.047 - Batch 103/249 - Min Loss:11.11 - Loss:11.112100526728414\n",
      " Iter:5 - Alpha:0.047 - Batch 104/249 - Min Loss:11.10 - Loss:11.104394523353339\n",
      " Iter:5 - Alpha:0.047 - Batch 105/249 - Min Loss:11.09 - Loss:11.099332194264191\n",
      " Iter:5 - Alpha:0.047 - Batch 107/249 - Min Loss:11.08 - Loss:11.084563410952635\n",
      " Iter:5 - Alpha:0.047 - Batch 128/249 - Min Loss:11.07 - Loss:11.080755098118637\n",
      " Iter:5 - Alpha:0.047 - Batch 129/249 - Min Loss:11.07 - Loss:11.074532810524575\n",
      " Iter:5 - Alpha:0.047 - Batch 130/249 - Min Loss:11.07 - Loss:11.070308307744044\n",
      " Iter:5 - Alpha:0.047 - Batch 131/249 - Min Loss:11.05 - Loss:11.053205895772722\n",
      " Iter:5 - Alpha:0.047 - Batch 133/249 - Min Loss:11.03 - Loss:11.041628774766117\n",
      " Iter:5 - Alpha:0.047 - Batch 135/249 - Min Loss:11.03 - Loss:11.035707041019423\n",
      " Iter:5 - Alpha:0.047 - Batch 137/249 - Min Loss:11.02 - Loss:11.024606766677806\n",
      " Iter:5 - Alpha:0.047 - Batch 138/249 - Min Loss:11.00 - Loss:11.007025857860137\n",
      " Iter:5 - Alpha:0.047 - Batch 144/249 - Min Loss:11.00 - Loss:11.004993617647228\n",
      " Iter:5 - Alpha:0.047 - Batch 152/249 - Min Loss:11.00 - Loss:11.003699896686872\n",
      " Iter:5 - Alpha:0.047 - Batch 153/249 - Min Loss:10.99 - Loss:10.996498599690339\n",
      " Iter:5 - Alpha:0.047 - Batch 208/249 - Min Loss:10.98 - Loss:10.995703693907218\n",
      " Iter:5 - Alpha:0.047 - Batch 209/249 - Min Loss:10.98 - Loss:10.987160856768003\n",
      " Iter:5 - Alpha:0.047 - Batch 210/249 - Min Loss:10.97 - Loss:10.979813712335138\n",
      " Iter:5 - Alpha:0.047 - Batch 211/249 - Min Loss:10.96 - Loss:10.961275932129482\n",
      " Iter:5 - Alpha:0.047 - Batch 212/249 - Min Loss:10.95 - Loss:10.954692498871907\n",
      " Iter:5 - Alpha:0.047 - Batch 213/249 - Min Loss:10.94 - Loss:10.948507347452539\n",
      " Iter:5 - Alpha:0.047 - Batch 214/249 - Min Loss:10.94 - Loss:10.947598863066297\n",
      " Iter:5 - Alpha:0.047 - Batch 215/249 - Min Loss:10.93 - Loss:10.937779129758908\n",
      " Iter:5 - Alpha:0.047 - Batch 217/249 - Min Loss:10.93 - Loss:10.933565142331665\n",
      " Iter:5 - Alpha:0.047 - Batch 218/249 - Min Loss:10.92 - Loss:10.927695951351394\n",
      " Iter:5 - Alpha:0.047 - Batch 223/249 - Min Loss:10.91 - Loss:10.920012740008673\n",
      " Iter:5 - Alpha:0.047 - Batch 224/249 - Min Loss:10.91 - Loss:10.915941202659539\n",
      " Iter:5 - Alpha:0.047 - Batch 225/249 - Min Loss:10.91 - Loss:10.911412416260603\n",
      " Iter:5 - Alpha:0.047 - Batch 226/249 - Min Loss:10.90 - Loss:10.908265904635522\n",
      " Iter:5 - Alpha:0.047 - Batch 235/249 - Min Loss:10.90 - Loss:10.905597365624304\n",
      " Iter:5 - Alpha:0.047 - Batch 236/249 - Min Loss:10.89 - Loss:10.898794557204706\n",
      " Iter:5 - Alpha:0.047 - Batch 237/249 - Min Loss:10.89 - Loss:10.890970313767603\n",
      " Iter:5 - Alpha:0.047 - Batch 240/249 - Min Loss:10.88 - Loss:10.88534610730771\n",
      " Iter:5 - Alpha:0.047 - Batch 241/249 - Min Loss:10.87 - Loss:10.876534714857252\n",
      " Iter:5 - Alpha:0.047 - Batch 242/249 - Min Loss:10.87 - Loss:10.87459117637956\n",
      " Iter:5 - Alpha:0.047 - Batch 248/249 - Min Loss:10.86 - Loss:10.872254639188432\n",
      " Iter:5 - Alpha:0.047 - Batch 249/249 - Min Loss:10.86 - Loss:10.86373688723013\n",
      " Iter:6 - Alpha:0.047 - Batch 1/249 - Min Loss:10.69 - Loss:10.690402702580894 - hen theres, and theres, and theres, and theres, and theres, and theres\n",
      " Iter:7 - Alpha:0.046 - Batch 140/249 - Min Loss:10.55 - Loss:10.736922784153954heres, and seent thees, and seent thees, and seent thees, and seent th"
     ]
    }
   ],
   "source": [
    "train(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Iter:91 - Alpha:0.016 - Batch 176/249 - Min Loss:9.900 - Loss:11.975722569949843\n"
     ]
    }
   ],
   "source": [
    "train(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intestay thee.\n",
      "\n",
      "SIR:\n",
      "It thou my thar the sentastar the see the see:\n",
      "Imentary take the subloud I\n",
      "Stall my thentaring fook the senternight pead me, the gakentlenternot they day them.\n",
      "\n",
      "KENNOR:\n",
      "I stay the see talk :\n",
      "Non the seady!\n",
      "\n",
      "Sustar thou shour in the suble the see the senternow the antently the see the seaventlace peake,\n",
      "I sentlentony my thent:\n",
      "I the sentastar thamy this not thame.\n",
      "\n",
      "From the stay the sentastar star the see the senternce thentlent\n",
      "stay you, he shad be his say the senterny astak\n"
     ]
    }
   ],
   "source": [
    "def generate_sample(n=30, init_char=' '):\n",
    "    s = \"\"\n",
    "    hidden = model.init_hidden(batch_size=1)\n",
    "    input = Tensor(np.array([word2index[init_char]]))\n",
    "    for i in range(n):\n",
    "        rnn_input = embed.forward(input)\n",
    "        output, hidden = model.forward(input=rnn_input, hidden=hidden)\n",
    "        output.data *= 15\n",
    "        temp_dist = output.softmax()\n",
    "        temp_dist /= temp_dist.sum()\n",
    "\n",
    "#         m = (temp_dist > np.random.rand()).argmax() # sample from predictions\n",
    "        m = output.data.argmax() # take the max prediction\n",
    "        c = vocab[m]\n",
    "        input = Tensor(np.array([m]))\n",
    "        s += c\n",
    "    return s\n",
    "print(generate_sample(n=500, init_char='\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}