GDA - LDA - BinaryClasst.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gaussian Discriminant Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For binary output $y \\in \\{ 0, 1 \\}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "dataset = load_iris()\n",
    "X = dataset.data\n",
    "y = dataset.target\n",
    "\n",
    "# Change to binary class\n",
    "y = (y > 0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GaussianDiscAnalysis: \n",
    "    \"\"\"\n",
    "    Fits a Linear Discriminant Analysis Model for binary class\n",
    "    \n",
    "    lamd : Preconditioner; for matrix inversion; default:1e-10\n",
    "    \"\"\"\n",
    "    def __init__(self, lambd=1e-10):\n",
    "        self.lambd = lambd\n",
    "        \n",
    "    def compute_phi(self, y):\n",
    "        return (1/len(y)) * len(y[y==1])\n",
    "    \n",
    "    def compute_mu(self, X, y, idx):\n",
    "        return np.sum(X[y==idx], axis=0)/ len(y==idx)\n",
    "    \n",
    "    \n",
    "    def compute_sigma(self, X, y):\n",
    "        n = len(X)\n",
    "        #y = y.reshape(-1,1)\n",
    "        mu1 = self.mu[1]\n",
    "        mu0 = self.mu[0]\n",
    "        Xmu = X \\\n",
    "              - mu1*np.ones_like(X)*(y==1).reshape(-1,1) \\\n",
    "              - mu0*np.ones_like(X)*(y==0).reshape(-1,1)\n",
    "        return (1/n) * Xmu.T@Xmu\n",
    "    \n",
    "    \n",
    "    def compute_Pxyi(self, X, idx):\n",
    "        \"\"\"Probability of X given y\"\"\"\n",
    "        d = X.shape[1]\n",
    "        sigma_inv = np.linalg.inv(self.sigma)\n",
    "        det_sigma = np.linalg.det(self.sigma)\n",
    "        #mu_i = mu(X, y, idx)\n",
    "        Pxi = (1/((2*np.pi)**(d/2))) \\\n",
    "                *(1/(det_sigma**0.5)) \\\n",
    "                * np.exp(- 0.5*np.sum(((X-self.mu[idx])@sigma_inv)*(X-self.mu[idx]), axis=1))\n",
    "    #     Pxi = np.log(1) \\\n",
    "    #             - np.log((2*np.pi)**(m/2)) \\\n",
    "    #             - np.log(np.sqrt(det_sigma)) \\\n",
    "    #             - np.sum(((X-mu_i)@sigma_inv)*(X-mu_i), axis=1)\n",
    "        return Pxi\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        \"\"\"Computes mean, covariance and proabilities of y (phi)\"\"\"\n",
    "        d = X.shape[1]\n",
    "        self.mu = []\n",
    "        for i in np.unique(y):\n",
    "            self.mu.append(self.compute_mu(X, y, i))\n",
    "        #self.mu1 = self.compute_mu(X, y, 1)\n",
    "        self.sigma = self.compute_sigma(X, y) + self.lambd*np.eye(d,d)\n",
    "        self.phi = self.compute_phi(y)\n",
    "        \n",
    "    def predict(self, X):\n",
    "        Py0 = self.compute_Pxyi(X, 0) * (1-self.phi)\n",
    "        Py1 = self.compute_Pxyi(X, 1) * self.phi\n",
    "        Py0 = Py0.reshape(-1, 1)\n",
    "        Py1 = Py1.reshape(-1, 1)\n",
    "        return np.argmax(np.concatenate((Py0, Py1), axis=1), axis=1)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GDA = GaussianDiscAnalysis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "GDA.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = GDA.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1,\n",
       "       0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(predictions == y_test) / len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}