Chapter8.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ensembles\"\n",
    "\n",
    "def image_path(fig_id):\n",
    "    return os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(image_path(fig_id) + \".png\", format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.01570027, -0.55091331, -0.26132626],\n",
       "       [-0.00771675,  0.59958572,  0.03507755],\n",
       "       [-0.95317135, -0.46453691, -0.24920288],\n",
       "       [-0.92012304,  0.21009593,  0.02182381],\n",
       "       [-0.76309739,  0.158261  ,  0.19152496],\n",
       "       [ 1.11816122,  0.32508721,  0.31710572],\n",
       "       [-1.02258878, -0.64384064, -0.13368695],\n",
       "       [ 0.67351984, -0.27342519, -0.00787835],\n",
       "       [ 1.01619558,  0.51546608,  0.46783297],\n",
       "       [ 0.54957723,  0.67728016,  0.2340159 ],\n",
       "       [-0.98960443,  0.00886617, -0.12152034],\n",
       "       [ 1.13248106,  0.28229967,  0.06972972],\n",
       "       [-0.99337712, -0.26832824, -0.15761101],\n",
       "       [-0.94763679, -0.4995849 , -0.13927911],\n",
       "       [ 1.10095709,  0.1706481 ,  0.09790432],\n",
       "       [-0.34604591,  0.45334414,  0.06433843],\n",
       "       [ 0.69102621, -0.27337761, -0.05926516],\n",
       "       [ 0.68302902,  0.70421846,  0.24642318],\n",
       "       [ 0.8774031 , -0.16775101, -0.02162333],\n",
       "       [-1.06090127, -0.48213721, -0.38573526],\n",
       "       [ 0.52336644,  0.66585845,  0.39019099],\n",
       "       [-0.94419403, -0.61502157, -0.47610118],\n",
       "       [-0.9716288 ,  0.00742468, -0.18764369],\n",
       "       [-1.10560661, -0.31903307, -0.17189644],\n",
       "       [ 1.1748694 ,  0.15718214,  0.26579776],\n",
       "       [ 0.91337123, -0.1608149 ,  0.02223257],\n",
       "       [-0.28688829,  0.56373778,  0.11210787],\n",
       "       [ 1.10247263,  0.21024786,  0.17826927],\n",
       "       [-0.93581594,  0.14433427, -0.07641252],\n",
       "       [ 0.6617217 ,  0.7350669 ,  0.37436649],\n",
       "       [ 0.09896169,  0.60619461,  0.14346668],\n",
       "       [-1.08103158, -0.54686117, -0.37070336],\n",
       "       [ 0.03004261,  0.60541538,  0.13736259],\n",
       "       [ 0.97846627, -0.04833433,  0.09297312],\n",
       "       [ 0.97257451,  0.15564902,  0.2668735 ],\n",
       "       [-0.06924261,  0.57029162,  0.32415098],\n",
       "       [ 0.01690169,  0.64656545,  0.15571838],\n",
       "       [-0.40544483,  0.39226596, -0.06928305],\n",
       "       [ 0.64162827,  0.68484229,  0.13122889],\n",
       "       [-0.64012055,  0.41578899,  0.27325686],\n",
       "       [ 0.7326402 ,  0.75604792,  0.25979142],\n",
       "       [-0.44999735,  0.52952899,  0.15964958],\n",
       "       [-0.98461065,  0.02344158,  0.1002999 ],\n",
       "       [ 1.11939222,  0.24629838,  0.11781711],\n",
       "       [ 0.78441066,  0.64312236,  0.25192037],\n",
       "       [ 0.16636498,  0.71958358,  0.17445205],\n",
       "       [-0.67371747,  0.3528853 , -0.07109776],\n",
       "       [ 1.11429663,  0.41882472,  0.31705263],\n",
       "       [-0.11381596,  0.6373893 ,  0.28640517],\n",
       "       [-0.45471725,  0.43269163, -0.10303808],\n",
       "       [-1.15790478, -0.39121007, -0.13498243],\n",
       "       [ 0.83494104,  0.66412252,  0.25913415],\n",
       "       [-0.46940298,  0.39639057,  0.0903395 ],\n",
       "       [ 1.14845951,  0.31874645,  0.23025679],\n",
       "       [ 0.24039921,  0.59890442,  0.07305738],\n",
       "       [ 1.10567699,  0.21166076,  0.14297884],\n",
       "       [-1.01152009, -0.449484  , -0.33508853],\n",
       "       [ 0.52519411,  0.64730821,  0.43823703],\n",
       "       [-1.09575714, -0.16906908,  0.05828838],\n",
       "       [ 0.58692443,  0.62026757,  0.25317605]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(4)\n",
    "m = 60\n",
    "w1, w2 = 0.1, 0.3\n",
    "noise = 0.1\n",
    "\n",
    "angles = np.random.rand(m) * 3 * np.pi / 2 - 0.5\n",
    "X = np.empty((m, 3))\n",
    "X[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * np.random.randn(m) / 2\n",
    "X[:, 1] = np.sin(angles) * 0.7 + noise * np.random.randn(m) / 2\n",
    "X[:, 2] = X[:, 0] * w1 + X[:, 1] * w2 + noise * np.random.randn(m)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_centered = X - X.mean(axis=0)\n",
    "U, s, V = np.linalg.svd(X_centered)\n",
    "c1 = V.T[:, 0]\n",
    "c2 = V.T[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.26203346, -0.42067648],\n",
       "       [ 0.08001485,  0.35272239],\n",
       "       [-1.17545763, -0.36085729],\n",
       "       [-0.89305601,  0.30862856],\n",
       "       [-0.73016287,  0.25404049],\n",
       "       [ 1.10436914, -0.20204953],\n",
       "       [-1.27265808, -0.46781247],\n",
       "       [ 0.44933007, -0.67736663],\n",
       "       [ 1.09356195,  0.04467792],\n",
       "       [ 0.66177325,  0.28651264],\n",
       "       [-1.04466138,  0.11244353],\n",
       "       [ 1.05932502, -0.31189109],\n",
       "       [-1.13761426, -0.14576655],\n",
       "       [-1.16044117, -0.36481599],\n",
       "       [ 1.00167625, -0.39422008],\n",
       "       [-0.2750406 ,  0.34391089],\n",
       "       [ 0.45624787, -0.69707573],\n",
       "       [ 0.79706574,  0.26870969],\n",
       "       [ 0.66924929, -0.65520024],\n",
       "       [-1.30679728, -0.37671343],\n",
       "       [ 0.6626586 ,  0.32706423],\n",
       "       [-1.25387588, -0.56043928],\n",
       "       [-1.04046987,  0.08727672],\n",
       "       [-1.26047729, -0.1571074 ],\n",
       "       [ 1.09786649, -0.38643428],\n",
       "       [ 0.7130973 , -0.64941523],\n",
       "       [-0.17786909,  0.43609071],\n",
       "       [ 1.02975735, -0.33747452],\n",
       "       [-0.94552283,  0.22833268],\n",
       "       [ 0.80994916,  0.33810729],\n",
       "       [ 0.20189175,  0.3514758 ],\n",
       "       [-1.34219411, -0.42415687],\n",
       "       [ 0.13599883,  0.37258632],\n",
       "       [ 0.8206931 , -0.55120835],\n",
       "       [ 0.90818634, -0.31869127],\n",
       "       [ 0.06703671,  0.42486148],\n",
       "       [ 0.13936893,  0.41906961],\n",
       "       [-0.37356775,  0.27320849],\n",
       "       [ 0.7312441 ,  0.23441131],\n",
       "       [-0.5230355 ,  0.46621776],\n",
       "       [ 0.86146183,  0.30212526],\n",
       "       [-0.33203239,  0.47352674],\n",
       "       [-0.99467436,  0.18342807],\n",
       "       [ 1.04520043, -0.32697207],\n",
       "       [ 0.87477048,  0.18062856],\n",
       "       [ 0.30457923,  0.43904343],\n",
       "       [-0.63685997,  0.32851826],\n",
       "       [ 1.1287259 , -0.11627335],\n",
       "       [ 0.03836205,  0.49036349],\n",
       "       [-0.41386843,  0.31734423],\n",
       "       [-1.32417938, -0.1944472 ],\n",
       "       [ 0.92968677,  0.18429606],\n",
       "       [-0.40274964,  0.34154025],\n",
       "       [ 1.11480941, -0.24138847],\n",
       "       [ 0.31915065,  0.27787663],\n",
       "       [ 1.02666316, -0.34676546],\n",
       "       [-1.24145806, -0.35049349],\n",
       "       [ 0.66770361,  0.32262317],\n",
       "       [-1.16397896,  0.03648137],\n",
       "       [ 0.68326064,  0.22756871]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W2 = V.T[:, :2]\n",
    "X2D = X_centered.dot(W2)\n",
    "X2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.26203346,  0.42067648],\n",
       "       [-0.08001485, -0.35272239],\n",
       "       [ 1.17545763,  0.36085729],\n",
       "       [ 0.89305601, -0.30862856],\n",
       "       [ 0.73016287, -0.25404049],\n",
       "       [-1.10436914,  0.20204953],\n",
       "       [ 1.27265808,  0.46781247],\n",
       "       [-0.44933007,  0.67736663],\n",
       "       [-1.09356195, -0.04467792],\n",
       "       [-0.66177325, -0.28651264],\n",
       "       [ 1.04466138, -0.11244353],\n",
       "       [-1.05932502,  0.31189109],\n",
       "       [ 1.13761426,  0.14576655],\n",
       "       [ 1.16044117,  0.36481599],\n",
       "       [-1.00167625,  0.39422008],\n",
       "       [ 0.2750406 , -0.34391089],\n",
       "       [-0.45624787,  0.69707573],\n",
       "       [-0.79706574, -0.26870969],\n",
       "       [-0.66924929,  0.65520024],\n",
       "       [ 1.30679728,  0.37671343],\n",
       "       [-0.6626586 , -0.32706423],\n",
       "       [ 1.25387588,  0.56043928],\n",
       "       [ 1.04046987, -0.08727672],\n",
       "       [ 1.26047729,  0.1571074 ],\n",
       "       [-1.09786649,  0.38643428],\n",
       "       [-0.7130973 ,  0.64941523],\n",
       "       [ 0.17786909, -0.43609071],\n",
       "       [-1.02975735,  0.33747452],\n",
       "       [ 0.94552283, -0.22833268],\n",
       "       [-0.80994916, -0.33810729],\n",
       "       [-0.20189175, -0.3514758 ],\n",
       "       [ 1.34219411,  0.42415687],\n",
       "       [-0.13599883, -0.37258632],\n",
       "       [-0.8206931 ,  0.55120835],\n",
       "       [-0.90818634,  0.31869127],\n",
       "       [-0.06703671, -0.42486148],\n",
       "       [-0.13936893, -0.41906961],\n",
       "       [ 0.37356775, -0.27320849],\n",
       "       [-0.7312441 , -0.23441131],\n",
       "       [ 0.5230355 , -0.46621776],\n",
       "       [-0.86146183, -0.30212526],\n",
       "       [ 0.33203239, -0.47352674],\n",
       "       [ 0.99467436, -0.18342807],\n",
       "       [-1.04520043,  0.32697207],\n",
       "       [-0.87477048, -0.18062856],\n",
       "       [-0.30457923, -0.43904343],\n",
       "       [ 0.63685997, -0.32851826],\n",
       "       [-1.1287259 ,  0.11627335],\n",
       "       [-0.03836205, -0.49036349],\n",
       "       [ 0.41386843, -0.31734423],\n",
       "       [ 1.32417938,  0.1944472 ],\n",
       "       [-0.92968677, -0.18429606],\n",
       "       [ 0.40274964, -0.34154025],\n",
       "       [-1.11480941,  0.24138847],\n",
       "       [-0.31915065, -0.27787663],\n",
       "       [-1.02666316,  0.34676546],\n",
       "       [ 1.24145806,  0.35049349],\n",
       "       [-0.66770361, -0.32262317],\n",
       "       [ 1.16397896, -0.03648137],\n",
       "       [-0.68326064, -0.22756871]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA(n_components = 2)\n",
    "X2D = pca.fit_transform(X)\n",
    "X2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.84248607  0.14631839]\n"
     ]
    }
   ],
   "source": [
    "print(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "pca.fit(X)\n",
    "cumsum = np.cumsum(pca.explained_variance_ratio_)\n",
    "d = np.argmax(cumsum >= 0.95) + 1\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.26203346,  0.42067648],\n",
       "       [-0.08001485, -0.35272239],\n",
       "       [ 1.17545763,  0.36085729],\n",
       "       [ 0.89305601, -0.30862856],\n",
       "       [ 0.73016287, -0.25404049],\n",
       "       [-1.10436914,  0.20204953],\n",
       "       [ 1.27265808,  0.46781247],\n",
       "       [-0.44933007,  0.67736663],\n",
       "       [-1.09356195, -0.04467792],\n",
       "       [-0.66177325, -0.28651264],\n",
       "       [ 1.04466138, -0.11244353],\n",
       "       [-1.05932502,  0.31189109],\n",
       "       [ 1.13761426,  0.14576655],\n",
       "       [ 1.16044117,  0.36481599],\n",
       "       [-1.00167625,  0.39422008],\n",
       "       [ 0.2750406 , -0.34391089],\n",
       "       [-0.45624787,  0.69707573],\n",
       "       [-0.79706574, -0.26870969],\n",
       "       [-0.66924929,  0.65520024],\n",
       "       [ 1.30679728,  0.37671343],\n",
       "       [-0.6626586 , -0.32706423],\n",
       "       [ 1.25387588,  0.56043928],\n",
       "       [ 1.04046987, -0.08727672],\n",
       "       [ 1.26047729,  0.1571074 ],\n",
       "       [-1.09786649,  0.38643428],\n",
       "       [-0.7130973 ,  0.64941523],\n",
       "       [ 0.17786909, -0.43609071],\n",
       "       [-1.02975735,  0.33747452],\n",
       "       [ 0.94552283, -0.22833268],\n",
       "       [-0.80994916, -0.33810729],\n",
       "       [-0.20189175, -0.3514758 ],\n",
       "       [ 1.34219411,  0.42415687],\n",
       "       [-0.13599883, -0.37258632],\n",
       "       [-0.8206931 ,  0.55120835],\n",
       "       [-0.90818634,  0.31869127],\n",
       "       [-0.06703671, -0.42486148],\n",
       "       [-0.13936893, -0.41906961],\n",
       "       [ 0.37356775, -0.27320849],\n",
       "       [-0.7312441 , -0.23441131],\n",
       "       [ 0.5230355 , -0.46621776],\n",
       "       [-0.86146183, -0.30212526],\n",
       "       [ 0.33203239, -0.47352674],\n",
       "       [ 0.99467436, -0.18342807],\n",
       "       [-1.04520043,  0.32697207],\n",
       "       [-0.87477048, -0.18062856],\n",
       "       [-0.30457923, -0.43904343],\n",
       "       [ 0.63685997, -0.32851826],\n",
       "       [-1.1287259 ,  0.11627335],\n",
       "       [-0.03836205, -0.49036349],\n",
       "       [ 0.41386843, -0.31734423],\n",
       "       [ 1.32417938,  0.1944472 ],\n",
       "       [-0.92968677, -0.18429606],\n",
       "       [ 0.40274964, -0.34154025],\n",
       "       [-1.11480941,  0.24138847],\n",
       "       [-0.31915065, -0.27787663],\n",
       "       [-1.02666316,  0.34676546],\n",
       "       [ 1.24145806,  0.35049349],\n",
       "       [-0.66770361, -0.32262317],\n",
       "       [ 1.16397896, -0.03648137],\n",
       "       [-0.68326064, -0.22756871]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca = PCA(n_components=0.95)\n",
    "X_reduced = pca.fit_transform(X)\n",
    "X_reduced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "mnist = fetch_mldata('MNIST original')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_mnist = mnist[\"data\"]\n",
    "y_mnist = mnist[\"target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components = 154)\n",
    "X_mnist_reduced = pca.fit_transform(X_mnist)\n",
    "X_mnist_recovered = pca.inverse_transform(X_mnist_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import IncrementalPCA\n",
    "\n",
    "n_batches = 100\n",
    "inc_pca = IncrementalPCA(n_components=154)\n",
    "for X_batch in np.array_split(X_mnist, n_batches):\n",
    "    inc_pca.partial_fit(X_batch)\n",
    "\n",
    "X_mnist_reduced = inc_pca.transform(X_mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnd_pca = PCA(n_components=154, svd_solver=\"randomized\")\n",
    "X_reduced = rnd_pca.fit_transform(X_mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import KernelPCA\n",
    "\n",
    "rbf_pca = KernelPCA(n_components = 2, kernel=\"rbf\", gamma=0.04)\n",
    "X_reduced = rbf_pca.fit_transform(X)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}