democodewiki.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#mini-demo\n",
    "#we will use an LSTM RNN in TensorFlow that learns from wiki text to generate new text\n",
    "#dataset https://metamind.io/research/the-wikitext-long-term-dependency-language-modeling-dataset/\n",
    "#100 million tokens extracted from articles on Wikipedia, MetaMind bought by Salesforce\n",
    "#big companies are buying AI startups like candy\n",
    "#we will use tensorflow, no keras. no one liner models, we're building it all with TF, \n",
    "#including the internals of the LSTM cell\n",
    "\n",
    "#It will output the test input string plus 500 generated characters.\n",
    "\n",
    "import numpy as np #vectorization\n",
    "import random #generate probability distribution \n",
    "import tensorflow as tf #ml\n",
    "import datetime #clock training time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('text length in number of characters:', 1290590)\n",
      "head of text:\n",
      " \n",
      " = Robert Boulter = \n",
      " \n",
      " Robert Boulter is an English film , television and theatre actor . He had a guest @-@ starring role on the television series The Bill in 2000 . This was followed by a starring role in the play Herons written by Simon Stephens , which was performed in 2001 at the Royal Court Theatre . He had a guest role in the television series Judge John Deed in 2002 . In 2004 Boulter landed a role as \" Craig \" in the episode \" Teddy 's Story \" of the television series The Long Firm ; he starred alongside actors Mark Strong and Derek Jacobi . He was cast in the 2005 theatre productions of the Philip Ridley play Mercury Fur , which was performed at the Drum Theatre in Plymouth and the Menier Chocolate Factory in London . He was directed by John Tiffany and starred alongside Ben Whishaw , Shane Zaza , Harry Kent , Fraser Ayres , Sophie Stanton and Dominic Hall . \n",
      " In 2006 , Boulter starred alongside Whishaw in the play Citizenship written by Mark Ravenhill . He appeared on a 20\n"
     ]
    }
   ],
   "source": [
    "#lets open the text\n",
    "#native python file read function\n",
    "text = open('wiki.test.raw').read()\n",
    "print('text length in number of characters:', len(text))\n",
    "\n",
    "print('head of text:')\n",
    "print(text[:1000]) #all tokenized words, stored in a list called text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('number of characters:', 178)\n",
      "['\\n', ' ', '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', ']', '^', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '\\x80', '\\x81', '\\x82', '\\x83', '\\x84', '\\x85', '\\x86', '\\x87', '\\x88', '\\x89', '\\x8a', '\\x8b', '\\x8c', '\\x8d', '\\x8e', '\\x8f', '\\x90', '\\x91', '\\x92', '\\x93', '\\x94', '\\x95', '\\x96', '\\x97', '\\x98', '\\x99', '\\x9b', '\\x9c', '\\x9d', '\\x9e', '\\x9f', '\\xa0', '\\xa1', '\\xa2', '\\xa3', '\\xa4', '\\xa5', '\\xa6', '\\xa7', '\\xa8', '\\xa9', '\\xaa', '\\xab', '\\xad', '\\xae', '\\xaf', '\\xb0', '\\xb1', '\\xb2', '\\xb3', '\\xb4', '\\xb5', '\\xb6', '\\xb7', '\\xb8', '\\xb9', '\\xba', '\\xbb', '\\xbc', '\\xbd', '\\xbe', '\\xbf', '\\xc2', '\\xc3', '\\xc4', '\\xc5', '\\xc7', '\\xc9', '\\xca', '\\xcb', '\\xcc', '\\xcd', '\\xce', '\\xcf', '\\xd0', '\\xd7', '\\xd8', '\\xd9', '\\xda', '\\xe0', '\\xe1', '\\xe2', '\\xe3', '\\xe4', '\\xe5', '\\xe6', '\\xe7', '\\xe8', '\\xe9']\n",
      "hello\n"
     ]
    }
   ],
   "source": [
    "#A set is an unordered collection with no duplicate elements.\n",
    "#conver back to list, sorts alphanumerically\n",
    "#list of all unique chars\n",
    "chars = sorted(list(set(text)))\n",
    "char_size = len(chars)\n",
    "print('number of characters:', char_size)\n",
    "print(chars)\n",
    "print('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Character to id, and id to character\n",
    "#dictionary that maps each character to a number and vice versa\n",
    "char2id = dict((c, i) for i, c in enumerate(chars))\n",
    "id2char = dict((i, c) for i, c in enumerate(chars))\n",
    "#print(id2char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Given a probability of each character, return a likely character, one-hot encoded\n",
    "#our prediction will give us an array of probabilities of each character\n",
    "#we'll pick the most likely and one-hot encode it\n",
    "def sample(prediction):\n",
    "    #Samples are uniformly distributed over the half-open interval \n",
    "    r = random.uniform(0,1)\n",
    "    #store prediction char\n",
    "    s = 0\n",
    "    #since length > indices starting at 0\n",
    "    char_id = len(prediction) - 1\n",
    "    #for each char prediction probabilty\n",
    "    for i in range(len(prediction)):\n",
    "        #assign it to S\n",
    "        s += prediction[i]\n",
    "        #check if probability greater than our randomly generated one\n",
    "        if s >= r:\n",
    "            #if it is, thats the likely next char\n",
    "            char_id = i\n",
    "            break\n",
    "    #dont try to rank, just differentiate\n",
    "    #initialize the vector\n",
    "    char_one_hot = np.zeros(shape=[char_size])\n",
    "    #that characters ID encoded\n",
    "    #https://image.slidesharecdn.com/latin-150313140222-conversion-gate01/95/representation-learning-of-vectors-of-words-and-phrases-5-638.jpg?cb=1426255492\n",
    "    char_one_hot[char_id] = 1.0\n",
    "    return char_one_hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  1.  0. ...,  0.  0.  0.]\n",
      " ..., \n",
      " [ 0.  1.  0. ...,  0.  0.  0.]\n",
      " [ 0.  1.  0. ...,  0.  0.  0.]\n",
      " [ 0.  1.  0. ...,  0.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "#vectorize our data to feed it into model\n",
    "\n",
    "len_per_section = 50\n",
    "skip = 2\n",
    "sections = []\n",
    "next_chars = []\n",
    "#fill sections list with chunks of text, every 2 characters create a new 50 \n",
    "#character long section\n",
    "#because we are generating it at a character level\n",
    "for i in range(0, len(text) - len_per_section, skip):\n",
    "    sections.append(text[i: i + len_per_section])\n",
    "    next_chars.append(text[i + len_per_section])\n",
    "#Vectorize input and output\n",
    "#matrix of section length by num of characters\n",
    "X = np.zeros((len(sections), len_per_section, char_size))\n",
    "#label column for all the character id's, still zero\n",
    "y = np.zeros((len(sections), char_size))\n",
    "#for each char in each section, convert each char to an ID\n",
    "#for each section convert the labels to ids \n",
    "for i, section in enumerate(sections):\n",
    "    for j, char in enumerate(section):\n",
    "        X[i, j, char2id[char]] = 1\n",
    "    y[i, char2id[next_chars[i]]] = 1\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n",
      "('training data size:', 645270)\n",
      "('approximate steps per epoch:', 1260)\n"
     ]
    }
   ],
   "source": [
    "#Batch size defines number of samples that going to be propagated through the network.\n",
    "#one epoch = one forward pass and one backward pass of all the training examples\n",
    "#batch size = the number of training examples in one forward/backward pass.\n",
    "#The higher the batch size, the more memory space you'll need.\n",
    "#if you have 1000 training examples, \n",
    "#and your batch size is 500, then it will take 2 iterations to complete 1 epoch.\n",
    "batch_size = 512\n",
    "#total iterations\n",
    "max_steps = 72001\n",
    "#how often to log?\n",
    "log_every = 100\n",
    "#how often to save?\n",
    "save_every = 6000\n",
    "#too few and underfitting\n",
    "#Underfitting occurs when there are too few neurons \n",
    "#in the hidden layers to adequately detect the signals in a complicated data set.\n",
    "#too many and overfitting\n",
    "hidden_nodes = 1024\n",
    "#starting text\n",
    "test_start = 'I am thinking that'\n",
    "#to save our model\n",
    "checkpoint_directory = 'ckpt'\n",
    "\n",
    "#Create a checkpoint directory\n",
    "if tf.gfile.Exists(checkpoint_directory):\n",
    "    tf.gfile.DeleteRecursively(checkpoint_directory)\n",
    "tf.gfile.MakeDirs(checkpoint_directory)\n",
    "\n",
    "print('training data size:', len(X))\n",
    "print('approximate steps per epoch:', int(len(X)/batch_size))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#build our model time\n",
    "#create computation graph\n",
    "graph = tf.Graph()\n",
    "#if multiple graphs, but none here jsut one\n",
    "with graph.as_default():\n",
    "    ###########\n",
    "    #Prep\n",
    "    ###########\n",
    "    #Variables and placeholders\n",
    "    #global_step refer to the number of batches seen by the graph. \n",
    "    #Everytime a batch is provided, the weights are updated in the \n",
    "    #direction that minimizes the loss. global_step just keeps track \n",
    "    #of the number of batches seen so far starts off as 0\n",
    "    global_step = tf.Variable(0)\n",
    "    \n",
    "    #data tensor shape feeding in sections\n",
    "    data = tf.placeholder(tf.float32, [batch_size, len_per_section, char_size])\n",
    "    #labels\n",
    "    labels = tf.placeholder(tf.float32, [batch_size, char_size])\n",
    "    \n",
    "    #An LSTM RNN (Long Short Term Memory), consists of 3 gates and an internal state, \n",
    "    #This enables the LSTM to capture long-term dependencies. \n",
    "    #http://suriyadeepan.github.io/2017-02-13-unfolding-rnn-2/\n",
    "    #lets build weights and biases for each of the 3 gates and then for the cell state\n",
    "    \n",
    "    #tf variables\n",
    "    #Since we need the weights and biases for our model. \n",
    "    #We could imagine treating these like additional inputs, \n",
    "    #but TensorFlow has an even better way to handle it: Variable\n",
    "    #A Variable is a modifiable tensor that lives in TensorFlow's graph of \n",
    "    #interacting operations. It can be used and even modified by the computation. \n",
    "    #For machine learning applications, one generally has the model parameters be Variables.\n",
    "    \n",
    "    #Prep LSTM Operation\n",
    "    #Input gate: weights for input, weights for previous output, and bias\n",
    "    \n",
    "    #tf truncated normal\n",
    "    #Outputs random values from a truncated normal distribution.\n",
    "    #The generated values follow a normal distribution with specified mean and \n",
    "    #standard deviation, except that values whose magnitude is more than 2 standard deviations\n",
    "    #from the mean are dropped and re-picked.\n",
    "    #basically randomly initialized values here\n",
    "    \n",
    "    #biases act as an anchor\n",
    "\n",
    "    w_ii = tf.Variable(tf.truncated_normal([char_size, hidden_nodes], -0.1, 0.1))\n",
    "    w_io = tf.Variable(tf.truncated_normal([hidden_nodes, hidden_nodes], -0.1, 0.1))\n",
    "    b_i = tf.Variable(tf.zeros([1, hidden_nodes]))\n",
    "    #Forget gate: weights for input, weights for previous output, and bias\n",
    "    w_fi = tf.Variable(tf.truncated_normal([char_size, hidden_nodes], -0.1, 0.1))\n",
    "    w_fo = tf.Variable(tf.truncated_normal([hidden_nodes, hidden_nodes], -0.1, 0.1))\n",
    "    b_f = tf.Variable(tf.zeros([1, hidden_nodes]))\n",
    "    #Output gate: weights for input, weights for previous output, and bias\n",
    "    w_oi = tf.Variable(tf.truncated_normal([char_size, hidden_nodes], -0.1, 0.1))\n",
    "    w_oo = tf.Variable(tf.truncated_normal([hidden_nodes, hidden_nodes], -0.1, 0.1))\n",
    "    b_o = tf.Variable(tf.zeros([1, hidden_nodes]))\n",
    "    #Memory cell: weights for input, weights for previous output, and bias\n",
    "    w_ci = tf.Variable(tf.truncated_normal([char_size, hidden_nodes], -0.1, 0.1))\n",
    "    w_co = tf.Variable(tf.truncated_normal([hidden_nodes, hidden_nodes], -0.1, 0.1))\n",
    "    b_c = tf.Variable(tf.zeros([1, hidden_nodes]))\n",
    "    \n",
    "    \n",
    "    #LSTM Cell\n",
    "    # given input, output, external state, it will return output and state\n",
    "    #output starts off empty, LSTM cell calculates it\n",
    "    \n",
    "    #Since, we have two kinds of states - the internal state ct \n",
    "    #and the (exposed) external state st, and since we need both of \n",
    "    #them for the subsequent sequential operations, we combine them \n",
    "    #into a tensor at each step, and pass them as input to the next \n",
    "    #step. This tensor is unpacked into st_1 and ct_1 at the beginning of each step.\n",
    "    \n",
    "    \n",
    "    def lstm(i, o, state):\n",
    "        \n",
    "        #these are all calculated seperately, no overlap until....\n",
    "        #(input * input weights) + (output * weights for previous output) + bias\n",
    "        input_gate = tf.sigmoid(tf.matmul(i, w_ii) + tf.matmul(o, w_io) + b_i)\n",
    "        #(input * forget weights) + (output * weights for previous output) + bias\n",
    "        forget_gate = tf.sigmoid(tf.matmul(i, w_fi) + tf.matmul(o, w_fo) + b_f)\n",
    "        #(input * output weights) + (output * weights for previous output) + bias\n",
    "        output_gate = tf.sigmoid(tf.matmul(i, w_oi) + tf.matmul(o, w_oo) + b_o)\n",
    "        #(input * internal state weights) + (output * weights for previous output) + bias\n",
    "        memory_cell = tf.sigmoid(tf.matmul(i, w_ci) + tf.matmul(o, w_co) + b_c)\n",
    "        \n",
    "        #...now! multiply forget gate * given state    +  input gate * hidden state\n",
    "        state = forget_gate * state + input_gate * memory_cell\n",
    "        #squash that state with tanh nonlin (Computes hyperbolic tangent of x element-wise)\n",
    "        #multiply by output\n",
    "        output = output_gate * tf.tanh(state)\n",
    "        #return \n",
    "        return output, state\n",
    "    \n",
    "    ###########\n",
    "    #Operation\n",
    "    ###########\n",
    "    #LSTM\n",
    "    #both start off as empty, LSTM will calculate this\n",
    "    output = tf.zeros([batch_size, hidden_nodes])\n",
    "    state = tf.zeros([batch_size, hidden_nodes])\n",
    "\n",
    "    #unrolled LSTM loop\n",
    "    #for each input set\n",
    "    for i in range(len_per_section):\n",
    "        #calculate state and output from LSTM\n",
    "        output, state = lstm(data[:, i, :], output, state)\n",
    "        #to start, \n",
    "        if i == 0:\n",
    "            #store initial output and labels\n",
    "            outputs_all_i = output\n",
    "            labels_all_i = data[:, i+1, :]\n",
    "        #for each new set, concat outputs and labels\n",
    "        elif i != len_per_section - 1:\n",
    "            #concatenates (combines) vectors along a dimension axis, not multiply\n",
    "            outputs_all_i = tf.concat(0, [outputs_all_i, output])\n",
    "            labels_all_i = tf.concat(0, [labels_all_i, data[:, i+1, :]])\n",
    "        else:\n",
    "            #final store\n",
    "            outputs_all_i = tf.concat(0, [outputs_all_i, output])\n",
    "            labels_all_i = tf.concat(0, [labels_all_i, labels])\n",
    "        \n",
    "    #Classifier\n",
    "    #The Classifier will only run after saved_output and saved_state were assigned.\n",
    "    \n",
    "    #calculate weight and bias values for the network\n",
    "    #generated randomly given a size and distribution\n",
    "    w = tf.Variable(tf.truncated_normal([hidden_nodes, char_size], -0.1, 0.1))\n",
    "    b = tf.Variable(tf.zeros([char_size]))\n",
    "    #Logits simply means that the function operates on the unscaled output \n",
    "    #of earlier layers and that the relative scale to understand the units \n",
    "    #is linear. It means, in particular, the sum of the inputs may not equal 1, \n",
    "    #that the values are not probabilities (you might have an input of 5).\n",
    "    logits = tf.matmul(outputs_all_i, w) + b\n",
    "    \n",
    "    #logits is our prediction outputs, lets compare it with our labels\n",
    "    #cross entropy since multiclass classification\n",
    "    #computes the cost for a softmax layer\n",
    "    #then Computes the mean of elements across dimensions of a tensor.\n",
    "    #average loss across all values\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, labels_all_i))\n",
    "\n",
    "    #Optimizer\n",
    "    #minimize loss with graident descent, learning rate 10,  keep track of batches\n",
    "    optimizer = tf.train.GradientDescentOptimizer(10.).minimize(loss, global_step=global_step)\n",
    "    \n",
    "    ###########\n",
    "    #Test\n",
    "    ###########\n",
    "    #test_data = tf.placeholder(tf.float32, shape=[1, char_size])\n",
    "    #test_output = tf.Variable(tf.zeros([1, hidden_nodes]))\n",
    "    #test_state = tf.Variable(tf.zeros([1, hidden_nodes]))\n",
    "    \n",
    "    #Reset at the beginning of each test\n",
    "    #reset_test_state = tf.group(test_output.assign(tf.zeros([1, hidden_nodes])), \n",
    "                                #test_state.assign(tf.zeros([1, hidden_nodes])))\n",
    "\n",
    "    #LSTM\n",
    "    #test_output, test_state = lstm(test_data, test_output, test_state)\n",
    "    #test_prediction = tf.nn.softmax(tf.matmul(test_output, w) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training loss at step 0: 5.15 (2017-03-21 15:46:13.367470)\n",
      "================================================================================\n",
      "I am thinking that        e                                                                                                                                  e                                                                                                                                                                                                                                                                                                                                                                        \n",
      "================================================================================\n",
      "training loss at step 10: 6.73 (2017-03-21 15:48:13.244305)\n",
      "training loss at step 20: 3.22 (2017-03-21 15:50:01.814675)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-455f4f11e89d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m             \u001b[0mbatch_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0moffset\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mto_add\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m             \u001b[0moffset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_add\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m10\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    962\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    965\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1015\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1019\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1001\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1002\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1003\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1004\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#timew to train the model, initialize a session with a graph\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    #standard init step\n",
    "    tf.global_variables_initializer().run()\n",
    "    offset = 0\n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    #for each training step\n",
    "    for step in range(max_steps):\n",
    "        \n",
    "        #starts off as 0\n",
    "        offset = offset % len(X)\n",
    "        \n",
    "        #calculate batch data and labels to feed model iteratively\n",
    "        if offset <= (len(X) - batch_size):\n",
    "            #first part\n",
    "            batch_data = X[offset: offset + batch_size]\n",
    "            batch_labels = y[offset: offset + batch_size]\n",
    "            offset += batch_size\n",
    "        #until when offset  = batch size, then we \n",
    "        else:\n",
    "            #last part\n",
    "            to_add = batch_size - (len(X) - offset)\n",
    "            batch_data = np.concatenate((X[offset: len(X)], X[0: to_add]))\n",
    "            batch_labels = np.concatenate((y[offset: len(X)], y[0: to_add]))\n",
    "            offset = to_add\n",
    "        \n",
    "        #optimize!!\n",
    "        _, training_loss = sess.run([optimizer, loss], feed_dict={data: batch_data, labels: batch_labels})\n",
    "        \n",
    "        if step % 10 == 0:\n",
    "            print('training loss at step %d: %.2f (%s)' % (step, training_loss, datetime.datetime.now()))\n",
    "\n",
    "            if step % save_every == 0:\n",
    "                saver.save(sess, checkpoint_directory + '/model', global_step=step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_start = 'I plan to make the world a better place '\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    #init graph, load model\n",
    "    tf.global_variables_initializer().run()\n",
    "    model = tf.train.latest_checkpoint(checkpoint_directory)\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, model)\n",
    "\n",
    "    #set input variable to generate chars from\n",
    "    reset_test_state.run() \n",
    "    test_generated = test_start\n",
    "\n",
    "    #for every char in the input sentennce\n",
    "    for i in range(len(test_start) - 1):\n",
    "        #initialize an empty char store\n",
    "        test_X = np.zeros((1, char_size))\n",
    "        #store it in id from\n",
    "        test_X[0, char2id[test_start[i]]] = 1.\n",
    "        #feed it to model, test_prediction is the output value\n",
    "        _ = sess.run(test_prediction, feed_dict={test_data: test_X})\n",
    "\n",
    "    \n",
    "    #where we store encoded char predictions\n",
    "    test_X = np.zeros((1, char_size))\n",
    "    test_X[0, char2id[test_start[-1]]] = 1.\n",
    "\n",
    "    #lets generate 500 characters\n",
    "    for i in range(500):\n",
    "        #get each prediction probability\n",
    "        prediction = test_prediction.eval({test_data: test_X})[0]\n",
    "        #one hot encode it\n",
    "        next_char_one_hot = sample(prediction)\n",
    "        #get the indices of the max values (highest probability)  and convert to char\n",
    "        next_char = id2char[np.argmax(next_char_one_hot)]\n",
    "        #add each char to the output text iteratively\n",
    "        test_generated += next_char\n",
    "        #update the \n",
    "        test_X = next_char_one_hot.reshape((1, char_size))\n",
    "\n",
    "    print(test_generated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}