simplest_task.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from NALU import NALU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create the simplest training possible which is the addition of 2 numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 10000\n",
    "test_size = 1000\n",
    "lr = 0.1\n",
    "x_tr = np.random.rand(train_size, 2)*5\n",
    "# The test data is out of the range of training data\n",
    "x_tst = 5 + np.random.rand(test_size, 2)*5\n",
    "y_tr = x_tr.sum(1, keepdims=True)\n",
    "y_tst = x_tst.sum(1, keepdims=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "X = tf.placeholder(tf.float32, shape=(None, 2))\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1))\n",
    "y_pred = NALU(2, 1, name = 'nalu')(X)\n",
    "loss = tf.losses.mean_squared_error(y, y_pred)\n",
    "optim = tf.train.RMSPropOptimizer(lr).minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 loss: 23.576784133911133\n",
      "Epoch: 10 loss: 5.664342880249023\n",
      "Epoch: 20 loss: 2.3301279544830322\n",
      "Epoch: 30 loss: 0.7819363474845886\n",
      "Epoch: 40 loss: 0.3641888201236725\n",
      "Epoch: 50 loss: 0.1925293356180191\n",
      "Epoch: 60 loss: 0.09686878323554993\n",
      "Epoch: 70 loss: 0.05198545381426811\n",
      "Epoch: 80 loss: 0.029626082628965378\n",
      "Epoch: 90 loss: 0.01741959899663925\n",
      "Epoch: 100 loss: 0.010430704802274704\n",
      "Epoch: 110 loss: 0.006322233006358147\n",
      "Epoch: 120 loss: 0.003867687191814184\n",
      "Epoch: 130 loss: 0.0023848428390920162\n",
      "Epoch: 140 loss: 0.0014812853187322617\n",
      "Epoch: 150 loss: 0.0009268847643397748\n",
      "Epoch: 160 loss: 0.0005845101550221443\n",
      "Epoch: 170 loss: 0.00037174849421717227\n",
      "Epoch: 180 loss: 0.00023866309493314475\n",
      "Epoch: 190 loss: 0.0001548149302834645\n",
      "Epoch: 200 loss: 0.00010158126679016277\n",
      "Epoch: 210 loss: 6.750692409696057e-05\n",
      "Epoch: 220 loss: 4.5533797674579546e-05\n",
      "Epoch: 230 loss: 3.1302799470722675e-05\n",
      "Epoch: 240 loss: 2.2114592866273597e-05\n",
      "Epoch: 250 loss: 1.6235433577094227e-05\n",
      "Epoch: 260 loss: 1.2497339412220754e-05\n",
      "Epoch: 270 loss: 1.0090971045428887e-05\n",
      "Epoch: 280 loss: 8.487853847327642e-06\n",
      "Epoch: 290 loss: 7.3637424975459e-06\n",
      "Test loss: 8.50147262099199e-06\n"
     ]
    }
   ],
   "source": [
    "epochs = 300\n",
    "with tf.Session() as sess:\n",
    "        \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "    for epoch in range(epochs):\n",
    "        sess.run(optim, feed_dict={X:x_tr,y:y_tr})\n",
    "        if epoch % 10 == 0:\n",
    "            l = sess.run(loss,feed_dict={X:x_tr,y:y_tr})\n",
    "            print(\"Epoch: {} loss: {}\".format(epoch, l))\n",
    "                \n",
    "    test_preds, test_loss = sess.run([y_pred,loss],feed_dict={X:x_tst,y:y_tst})\n",
    "    print(\"Test loss: {}\".format(test_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After 200 epochs the training seems successful, let's try subtraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 10000\n",
    "test_size = 1000\n",
    "lr = 0.1\n",
    "x_tr = np.random.rand(train_size, 2)*5\n",
    "# The test data is out of the range of training data\n",
    "x_tst = 5 + np.random.rand(test_size, 2)*5\n",
    "y_tr = x_tr[:, 0:1]-x_tr[:, 1:2]\n",
    "y_tst = x_tst[:, 0:1]-x_tst[:, 1:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "X = tf.placeholder(tf.float32, shape=(None, 2))\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1))\n",
    "y_pred = NALU(2, 1, name = 'nalu')(X)\n",
    "loss = tf.losses.mean_squared_error(y, y_pred)\n",
    "optim = tf.train.RMSPropOptimizer(lr).minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 loss: 4.899702548980713\n",
      "Epoch: 10 loss: 1.7696599960327148\n",
      "Epoch: 20 loss: 0.7605103850364685\n",
      "Epoch: 30 loss: 0.3905770182609558\n",
      "Epoch: 40 loss: 0.210342675447464\n",
      "Epoch: 50 loss: 0.11742085218429565\n",
      "Epoch: 60 loss: 0.06794705986976624\n",
      "Epoch: 70 loss: 0.0411081425845623\n",
      "Epoch: 80 loss: 0.026101520285010338\n",
      "Epoch: 90 loss: 0.017384221777319908\n",
      "Epoch: 100 loss: 0.01210545003414154\n",
      "Epoch: 110 loss: 0.008766887709498405\n",
      "Epoch: 120 loss: 0.006558621302247047\n",
      "Epoch: 130 loss: 0.0050314306281507015\n",
      "Epoch: 140 loss: 0.003931527491658926\n",
      "Epoch: 150 loss: 0.0031137214973568916\n",
      "Epoch: 160 loss: 0.0024932322558015585\n",
      "Epoch: 170 loss: 0.002017725259065628\n",
      "Epoch: 180 loss: 0.0016518350457772613\n",
      "Epoch: 190 loss: 0.0013695494271814823\n",
      "Epoch: 200 loss: 0.0011509251780807972\n",
      "Epoch: 210 loss: 0.0009806206217035651\n",
      "Epoch: 220 loss: 0.0008469556923955679\n",
      "Epoch: 230 loss: 0.0007411165861412883\n",
      "Epoch: 240 loss: 0.0006564686773344874\n",
      "Epoch: 250 loss: 0.0005880054668523371\n",
      "Epoch: 260 loss: 0.0005319381016306579\n",
      "Epoch: 270 loss: 0.00048541068099439144\n",
      "Epoch: 280 loss: 0.0004462725773919374\n",
      "Epoch: 290 loss: 0.00041290654917247593\n",
      "Test loss: 3.3662645364529453e-06\n"
     ]
    }
   ],
   "source": [
    "epochs = 300\n",
    "with tf.Session() as sess:\n",
    "        \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "    for epoch in range(epochs):\n",
    "        sess.run(optim, feed_dict={X:x_tr,y:y_tr})\n",
    "        if epoch % 10 == 0:\n",
    "            l = sess.run(loss,feed_dict={X:x_tr,y:y_tr})\n",
    "            print(\"Epoch: {} loss: {}\".format(epoch, l))\n",
    "                \n",
    "    test_preds, test_loss = sess.run([y_pred,loss],feed_dict={X:x_tst,y:y_tst})\n",
    "    print(\"Test loss: {}\".format(test_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Good! Let's try multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 10000\n",
    "test_size = 1000\n",
    "lr = 0.1\n",
    "x_tr = np.random.rand(train_size, 2)*5\n",
    "# The test data is out of the range of training data\n",
    "x_tst = 5 + np.random.rand(test_size, 2)*5\n",
    "y_tr = x_tr.prod(1, keepdims=True)\n",
    "y_tst = x_tst.prod(1, keepdims=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "X = tf.placeholder(tf.float32, shape=(None, 2))\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1))\n",
    "y_pred = NALU(2, 1, name = 'nalu')(X)\n",
    "loss = tf.losses.mean_squared_error(y, y_pred)\n",
    "optim = tf.train.RMSPropOptimizer(lr).minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 loss: 61.78670120239258\n",
      "Epoch: 10 loss: 32.54534912109375\n",
      "Epoch: 20 loss: 7.915299892425537\n",
      "Epoch: 30 loss: 2.886151075363159\n",
      "Epoch: 40 loss: 1.335237741470337\n",
      "Epoch: 50 loss: 0.6910502910614014\n",
      "Epoch: 60 loss: 0.37979820370674133\n",
      "Epoch: 70 loss: 0.21625183522701263\n",
      "Epoch: 80 loss: 0.12591062486171722\n",
      "Epoch: 90 loss: 0.07441279292106628\n",
      "Epoch: 100 loss: 0.044440917670726776\n",
      "Epoch: 110 loss: 0.026747209951281548\n",
      "Epoch: 120 loss: 0.01619315892457962\n",
      "Epoch: 130 loss: 0.009849894791841507\n",
      "Epoch: 140 loss: 0.006014932878315449\n",
      "Epoch: 150 loss: 0.0036859558895230293\n",
      "Epoch: 160 loss: 0.002266236813738942\n",
      "Epoch: 170 loss: 0.0013980407966300845\n",
      "Epoch: 180 loss: 0.0008658133447170258\n",
      "Epoch: 190 loss: 0.0005384694668464363\n",
      "Epoch: 200 loss: 0.0003365428128745407\n",
      "Epoch: 210 loss: 0.00021156916045583785\n",
      "Epoch: 220 loss: 0.00013401758042164147\n",
      "Epoch: 230 loss: 8.572585647925735e-05\n",
      "Epoch: 240 loss: 5.561679063248448e-05\n",
      "Epoch: 250 loss: 3.685006595333107e-05\n",
      "Epoch: 260 loss: 2.5152505259029567e-05\n",
      "Epoch: 270 loss: 1.786893881217111e-05\n",
      "Epoch: 280 loss: 1.3320757716428488e-05\n",
      "Epoch: 290 loss: 1.0457918506290298e-05\n",
      "Test loss: 0.00032408267725259066\n"
     ]
    }
   ],
   "source": [
    "epochs = 300\n",
    "with tf.Session() as sess:\n",
    "        \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "    for epoch in range(epochs):\n",
    "        sess.run(optim, feed_dict={X:x_tr,y:y_tr})\n",
    "        if epoch % 10 == 0:\n",
    "            l = sess.run(loss,feed_dict={X:x_tr,y:y_tr})\n",
    "            print(\"Epoch: {} loss: {}\".format(epoch, l))\n",
    "                \n",
    "    test_preds, test_loss = sess.run([y_pred,loss],feed_dict={X:x_tst,y:y_tst})\n",
    "    print(\"Test loss: {}\".format(test_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This seems like a success!\n",
    "Let's try division"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 10000\n",
    "test_size = 1000\n",
    "lr = 0.1\n",
    "x_tr = np.random.rand(train_size, 2)*5\n",
    "# The test data is out of the range of training data\n",
    "x_tst = 5 + np.random.rand(test_size, 2)*5\n",
    "y_tr = x_tr[:, 0:1]/x_tr[:, 1:2]\n",
    "y_tst = x_tst[:, 0:1]/x_tst[:, 1:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "X = tf.placeholder(tf.float32, shape=(None, 2))\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1))\n",
    "y_pred = NALU(2, 1, name = 'nalu')(X)\n",
    "loss = tf.losses.mean_squared_error(y, y_pred)\n",
    "optim = tf.train.RMSPropOptimizer(lr).minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 loss: 8514.669921875\n",
      "Epoch: 10 loss: 3854.375244140625\n",
      "Epoch: 20 loss: 933.1788940429688\n",
      "Epoch: 30 loss: 376.2813415527344\n",
      "Epoch: 40 loss: 184.29025268554688\n",
      "Epoch: 50 loss: 98.67192840576172\n",
      "Epoch: 60 loss: 55.40634536743164\n",
      "Epoch: 70 loss: 31.995372772216797\n",
      "Epoch: 80 loss: 18.80275535583496\n",
      "Epoch: 90 loss: 11.180618286132812\n",
      "Epoch: 100 loss: 6.703417778015137\n",
      "Epoch: 110 loss: 4.044529914855957\n",
      "Epoch: 120 loss: 2.4528706073760986\n",
      "Epoch: 130 loss: 1.4945367574691772\n",
      "Epoch: 140 loss: 0.9143832325935364\n",
      "Epoch: 150 loss: 0.561928391456604\n",
      "Epoch: 160 loss: 0.3467443883419037\n",
      "Epoch: 170 loss: 0.2149953842163086\n",
      "Epoch: 180 loss: 0.13400445878505707\n",
      "Epoch: 190 loss: 0.0841434970498085\n",
      "Epoch: 200 loss: 0.053236883133649826\n",
      "Epoch: 210 loss: 0.03398091718554497\n",
      "Epoch: 220 loss: 0.02194857783615589\n",
      "Epoch: 230 loss: 0.014370208606123924\n",
      "Epoch: 240 loss: 0.009560810402035713\n",
      "Epoch: 250 loss: 0.006474270019680262\n",
      "Epoch: 260 loss: 0.004478793125599623\n",
      "Epoch: 270 loss: 0.003162340261042118\n",
      "Epoch: 280 loss: 0.00229236320592463\n",
      "Epoch: 290 loss: 0.0017081594560295343\n",
      "Test loss: 3.847848775717644e-11\n"
     ]
    }
   ],
   "source": [
    "epochs = 300\n",
    "with tf.Session() as sess:\n",
    "        \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "    for epoch in range(epochs):\n",
    "        sess.run(optim, feed_dict={X:x_tr,y:y_tr})\n",
    "        if epoch % 10 == 0:\n",
    "            l = sess.run(loss,feed_dict={X:x_tr,y:y_tr})\n",
    "            print(\"Epoch: {} loss: {}\".format(epoch, l))\n",
    "                \n",
    "    test_preds, test_loss = sess.run([y_pred,loss],feed_dict={X:x_tst,y:y_tst})\n",
    "    print(\"Test loss: {}\".format(test_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Success!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}